@yuihub/server 1.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/dist/api/text-process.d.ts +2 -0
- package/dist/api/text-process.d.ts.map +1 -0
- package/dist/api/text-process.js +48 -0
- package/dist/api/text-process.js.map +1 -0
- package/dist/auth.d.ts +30 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +71 -0
- package/dist/auth.js.map +1 -0
- package/dist/cli/setup.d.ts +2 -0
- package/dist/cli/setup.d.ts.map +1 -0
- package/dist/cli/setup.js +120 -0
- package/dist/cli/setup.js.map +1 -0
- package/dist/config/schema.d.ts +334 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +45 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/service.d.ts +12 -0
- package/dist/config/service.d.ts.map +1 -0
- package/dist/config/service.js +81 -0
- package/dist/config/service.js.map +1 -0
- package/dist/engine/agent/context.d.ts +35 -0
- package/dist/engine/agent/context.d.ts.map +1 -0
- package/dist/engine/agent/context.js +72 -0
- package/dist/engine/agent/context.js.map +1 -0
- package/dist/engine/agent/core.d.ts +19 -0
- package/dist/engine/agent/core.d.ts.map +1 -0
- package/dist/engine/agent/core.js +67 -0
- package/dist/engine/agent/core.js.map +1 -0
- package/dist/engine/agent/live-context.d.ts +8 -0
- package/dist/engine/agent/live-context.d.ts.map +1 -0
- package/dist/engine/agent/live-context.js +19 -0
- package/dist/engine/agent/live-context.js.map +1 -0
- package/dist/engine/agent/tools/fs.d.ts +16 -0
- package/dist/engine/agent/tools/fs.d.ts.map +1 -0
- package/dist/engine/agent/tools/fs.js +78 -0
- package/dist/engine/agent/tools/fs.js.map +1 -0
- package/dist/engine/agent/tools/memory.d.ts +18 -0
- package/dist/engine/agent/tools/memory.d.ts.map +1 -0
- package/dist/engine/agent/tools/memory.js +51 -0
- package/dist/engine/agent/tools/memory.js.map +1 -0
- package/dist/engine/ai/local-genai-service.d.ts +22 -0
- package/dist/engine/ai/local-genai-service.d.ts.map +1 -0
- package/dist/engine/ai/local-genai-service.js +158 -0
- package/dist/engine/ai/local-genai-service.js.map +1 -0
- package/dist/engine/ai/registry.d.ts +16 -0
- package/dist/engine/ai/registry.d.ts.map +1 -0
- package/dist/engine/ai/registry.js +78 -0
- package/dist/engine/ai/registry.js.map +1 -0
- package/dist/engine/ai/tools.d.ts +7 -0
- package/dist/engine/ai/tools.d.ts.map +1 -0
- package/dist/engine/ai/tools.js +2 -0
- package/dist/engine/ai/tools.js.map +1 -0
- package/dist/engine/ai/types.d.ts +21 -0
- package/dist/engine/ai/types.d.ts.map +1 -0
- package/dist/engine/ai/types.js +2 -0
- package/dist/engine/ai/types.js.map +1 -0
- package/dist/engine/ai/vertex-genai-service.d.ts +13 -0
- package/dist/engine/ai/vertex-genai-service.d.ts.map +1 -0
- package/dist/engine/ai/vertex-genai-service.js +93 -0
- package/dist/engine/ai/vertex-genai-service.js.map +1 -0
- package/dist/engine/chunker.d.ts +19 -0
- package/dist/engine/chunker.d.ts.map +1 -0
- package/dist/engine/chunker.js +79 -0
- package/dist/engine/chunker.js.map +1 -0
- package/dist/engine/composite-vector-store.d.ts +21 -0
- package/dist/engine/composite-vector-store.d.ts.map +1 -0
- package/dist/engine/composite-vector-store.js +69 -0
- package/dist/engine/composite-vector-store.js.map +1 -0
- package/dist/engine/embeddings/local-service.d.ts +10 -0
- package/dist/engine/embeddings/local-service.d.ts.map +1 -0
- package/dist/engine/embeddings/local-service.js +37 -0
- package/dist/engine/embeddings/local-service.js.map +1 -0
- package/dist/engine/embeddings/types.d.ts +10 -0
- package/dist/engine/embeddings/types.d.ts.map +1 -0
- package/dist/engine/embeddings/types.js +2 -0
- package/dist/engine/embeddings/types.js.map +1 -0
- package/dist/engine/embeddings/vertex-service.d.ts +11 -0
- package/dist/engine/embeddings/vertex-service.d.ts.map +1 -0
- package/dist/engine/embeddings/vertex-service.js +45 -0
- package/dist/engine/embeddings/vertex-service.js.map +1 -0
- package/dist/engine/indexer.d.ts +12 -0
- package/dist/engine/indexer.d.ts.map +1 -0
- package/dist/engine/indexer.js +74 -0
- package/dist/engine/indexer.js.map +1 -0
- package/dist/engine/lock.d.ts +24 -0
- package/dist/engine/lock.d.ts.map +1 -0
- package/dist/engine/lock.js +58 -0
- package/dist/engine/lock.js.map +1 -0
- package/dist/engine/schema.d.ts +19 -0
- package/dist/engine/schema.d.ts.map +1 -0
- package/dist/engine/schema.js +18 -0
- package/dist/engine/schema.js.map +1 -0
- package/dist/engine/vector-store-types.d.ts +24 -0
- package/dist/engine/vector-store-types.d.ts.map +1 -0
- package/dist/engine/vector-store-types.js +2 -0
- package/dist/engine/vector-store-types.js.map +1 -0
- package/dist/engine/vector-store.d.ts +21 -0
- package/dist/engine/vector-store.d.ts.map +1 -0
- package/dist/engine/vector-store.js +105 -0
- package/dist/engine/vector-store.js.map +1 -0
- package/dist/engine/watcher.d.ts +16 -0
- package/dist/engine/watcher.d.ts.map +1 -0
- package/dist/engine/watcher.js +94 -0
- package/dist/engine/watcher.js.map +1 -0
- package/dist/server.d.ts +2 -0
- package/dist/server.d.ts.map +1 -0
- package/dist/server.js +360 -0
- package/dist/server.js.map +1 -0
- package/dist/sync/github-provider.d.ts +20 -0
- package/dist/sync/github-provider.d.ts.map +1 -0
- package/dist/sync/github-provider.js +80 -0
- package/dist/sync/github-provider.js.map +1 -0
- package/dist/sync/scheduler.d.ts +18 -0
- package/dist/sync/scheduler.d.ts.map +1 -0
- package/dist/sync/scheduler.js +67 -0
- package/dist/sync/scheduler.js.map +1 -0
- package/package.json +78 -0
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* CompositeVectorStore
|
|
3
|
+
* Manages multiple VectorStore instances (Dual Embedding).
|
|
4
|
+
* - Writes to ALL stores (Fan-out).
|
|
5
|
+
* - Searches from ALL stores and merges using RRF.
|
|
6
|
+
*/
|
|
7
|
+
export class CompositeVectorStore {
|
|
8
|
+
stores;
|
|
9
|
+
constructor(stores) {
|
|
10
|
+
this.stores = stores;
|
|
11
|
+
}
|
|
12
|
+
async init() {
|
|
13
|
+
await Promise.all(this.stores.map(s => s.init()));
|
|
14
|
+
}
|
|
15
|
+
async add(entries) {
|
|
16
|
+
// Fan-out write with error collection
|
|
17
|
+
const errors = [];
|
|
18
|
+
await Promise.all(this.stores.map(async (s) => {
|
|
19
|
+
try {
|
|
20
|
+
await s.add(entries);
|
|
21
|
+
}
|
|
22
|
+
catch (e) {
|
|
23
|
+
console.error(`[CompositeVectorStore] add failed for store:`, e);
|
|
24
|
+
errors.push(e);
|
|
25
|
+
}
|
|
26
|
+
}));
|
|
27
|
+
// If all stores failed, throw
|
|
28
|
+
if (errors.length === this.stores.length) {
|
|
29
|
+
throw new Error('All stores failed to add entries');
|
|
30
|
+
}
|
|
31
|
+
}
|
|
32
|
+
async isEmpty() {
|
|
33
|
+
// Return true if all stores are empty
|
|
34
|
+
const results = await Promise.all(this.stores.map(s => s.isEmpty()));
|
|
35
|
+
return results.every(r => r === true);
|
|
36
|
+
}
|
|
37
|
+
async deleteBySource(source) {
|
|
38
|
+
// Fan-out delete and sum results
|
|
39
|
+
const counts = await Promise.all(this.stores.map(s => s.deleteBySource(source)));
|
|
40
|
+
return counts.reduce((sum, c) => sum + c, 0);
|
|
41
|
+
}
|
|
42
|
+
async search(query, limit = 10, filter) {
|
|
43
|
+
// Hybrid Search: RRF
|
|
44
|
+
// 1. Parallel Search
|
|
45
|
+
const resultsPerStore = await Promise.all(this.stores.map(s => s.search(query, limit * 2, filter))); // Fetch deeper for fusing
|
|
46
|
+
// 2. RRF Fusion
|
|
47
|
+
// RRF score = sum(1 / (k + rank))
|
|
48
|
+
const K = 60;
|
|
49
|
+
const scores = new Map();
|
|
50
|
+
const docMap = new Map(); // Keep detailed doc
|
|
51
|
+
for (const results of resultsPerStore) {
|
|
52
|
+
results.forEach((doc, rank) => {
|
|
53
|
+
if (!docMap.has(doc.id)) {
|
|
54
|
+
docMap.set(doc.id, doc);
|
|
55
|
+
}
|
|
56
|
+
const currentScore = scores.get(doc.id) || 0;
|
|
57
|
+
const rrfContribution = 1.0 / (K + rank + 1);
|
|
58
|
+
scores.set(doc.id, currentScore + rrfContribution);
|
|
59
|
+
});
|
|
60
|
+
}
|
|
61
|
+
// 3. Sort & Limit
|
|
62
|
+
const sortedIds = Array.from(scores.entries())
|
|
63
|
+
.sort((a, b) => b[1] - a[1]) // Descending score
|
|
64
|
+
.slice(0, limit)
|
|
65
|
+
.map(([id]) => id);
|
|
66
|
+
return sortedIds.map(id => docMap.get(id));
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
//# sourceMappingURL=composite-vector-store.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"composite-vector-store.js","sourceRoot":"","sources":["../../src/engine/composite-vector-store.ts"],"names":[],"mappings":"AAGA;;;;;GAKG;AACH,MAAM,OAAO,oBAAoB;IACvB,MAAM,CAAiB;IAE/B,YAAY,MAAsB;QAChC,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,GAAG,CAAC,OAAgB;QACxB,sCAAsC;QACtC,MAAM,MAAM,GAAY,EAAE,CAAC;QAC3B,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,EAAC,CAAC,EAAC,EAAE;YAC1C,IAAI,CAAC;gBACH,MAAM,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACvB,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,OAAO,CAAC,KAAK,CAAC,8CAA8C,EAAE,CAAC,CAAC,CAAC;gBACjE,MAAM,CAAC,IAAI,CAAC,CAAU,CAAC,CAAC;YAC1B,CAAC;QACH,CAAC,CAAC,CAAC,CAAC;QACJ,8BAA8B;QAC9B,IAAI,MAAM,CAAC,MAAM,KAAK,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;YACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,CAAC,CAAC;QACtD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,OAAO;QACX,sCAAsC;QACtC,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;QACrE,OAAO,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC;IACxC,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,MAAc;QACjC,iCAAiC;QACjC,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,KAAa,EAAE,QAAgB,EAAE,EAAE,MAA2C;QACzF,qBAAqB;QACrB,qBAAqB;QACrB,MAAM,eAAe,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,GAAG,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,0BAA0B;QAE/H,gBAAgB;QAChB,kCAAkC;QAClC,MAAM,CAAC,GAAG,EAAE,CAAC;QACb,MAAM,MAAM,GAAwB,IAAI,GAAG,EAAE,CAAC;QAC9C,MAAM,MAAM,GAA8B,IAAI,GAAG,EAAE,CAAC,CAAC,oBAAoB;QAEzE,KAAK,MAAM,OAAO,IAAI,eAAe,EAAE,CAAC;YACtC,OAAO,CAAC,OAAO,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,EAAE;gBAC5B,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC;oBACtB,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;gBAC5B,CAAC;gBACD,MAAM,YAAY,GAAG,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,CAAC;gBAC7C,MAAM,eAAe,GAAG,GAAG,GAAG,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,CAAC;gBAC7C,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,GAAG,eAAe,CAAC,CAAC;YACrD,CAAC,CAAC,CAAC;QACL,CAAC;QAED,kBAAkB;QAClB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;aAC3C,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,mBAAmB;aAC/C,KAAK,CAAC,CAAC,EAAE,KAAK,CAAC;aACf,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC,CAAC;QAErB,OAAO,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAE,CAAC,CAAC;IAC9C,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import { IEmbeddingService, EmbeddingOutput } from './types.js';
|
|
2
|
+
export declare class LocalEmbeddingService implements IEmbeddingService {
|
|
3
|
+
private embedder;
|
|
4
|
+
private modelName;
|
|
5
|
+
constructor(modelName?: string);
|
|
6
|
+
init(): Promise<void>;
|
|
7
|
+
embed(text: string): Promise<EmbeddingOutput>;
|
|
8
|
+
getDimensions(): number;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=local-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-service.d.ts","sourceRoot":"","sources":["../../../src/engine/embeddings/local-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAEhE,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,OAAO,CAAC,QAAQ,CAAa;IAC7B,OAAO,CAAC,SAAS,CAA2B;gBAEhC,SAAS,CAAC,EAAE,MAAM;IAIxB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAMrB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAYnD,aAAa,IAAI,MAAM;CASxB"}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { pipeline } from '@xenova/transformers';
|
|
2
|
+
export class LocalEmbeddingService {
|
|
3
|
+
embedder = null;
|
|
4
|
+
modelName = 'Xenova/bge-m3';
|
|
5
|
+
constructor(modelName) {
|
|
6
|
+
if (modelName)
|
|
7
|
+
this.modelName = modelName;
|
|
8
|
+
}
|
|
9
|
+
async init() {
|
|
10
|
+
if (this.embedder)
|
|
11
|
+
return;
|
|
12
|
+
// NOTE: This downloads the model on first run.
|
|
13
|
+
this.embedder = await pipeline('feature-extraction', this.modelName);
|
|
14
|
+
}
|
|
15
|
+
async embed(text) {
|
|
16
|
+
if (!this.embedder)
|
|
17
|
+
throw new Error('Embedder not initialized');
|
|
18
|
+
// Pooling: mean, Normalize: true is standard for sentence similarity
|
|
19
|
+
const output = await this.embedder(text, { pooling: 'mean', normalize: true });
|
|
20
|
+
return {
|
|
21
|
+
data: Array.from(output.data),
|
|
22
|
+
dimensions: output.dims ? output.dims[1] : output.data.length
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
getDimensions() {
|
|
26
|
+
// Default for all-MiniLM-L6-v2 is 384.
|
|
27
|
+
// If using bge-m3, it is 1024.
|
|
28
|
+
// Ideally this should be determined after init, but for now we hardcode or guess based on model name?
|
|
29
|
+
// Or we fetch it from embedder config if possible.
|
|
30
|
+
if (this.modelName.includes('MiniLM'))
|
|
31
|
+
return 384;
|
|
32
|
+
if (this.modelName.includes('bge-m3'))
|
|
33
|
+
return 1024;
|
|
34
|
+
return 384; // Fallback
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
//# sourceMappingURL=local-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"local-service.js","sourceRoot":"","sources":["../../../src/engine/embeddings/local-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,sBAAsB,CAAC;AAGhD,MAAM,OAAO,qBAAqB;IACxB,QAAQ,GAAQ,IAAI,CAAC;IACrB,SAAS,GAAW,eAAe,CAAC;IAE5C,YAAY,SAAkB;QAC5B,IAAI,SAAS;YAAE,IAAI,CAAC,SAAS,GAAG,SAAS,CAAC;IAC5C,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,IAAI,CAAC,QAAQ;YAAE,OAAO;QAC1B,+CAA+C;QAC/C,IAAI,CAAC,QAAQ,GAAG,MAAM,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,CAAC,CAAC;IACvE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAC;QAEhE,qEAAqE;QACrE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAE/E,OAAO;YACL,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;YAC7B,UAAU,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM;SAC9D,CAAC;IACJ,CAAC;IAED,aAAa;QACX,wCAAwC;QACxC,+BAA+B;QAC/B,sGAAsG;QACtG,mDAAmD;QACnD,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,GAAG,CAAC;QAClD,IAAI,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAC;QACnD,OAAO,GAAG,CAAC,CAAC,WAAW;IACzB,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
export interface EmbeddingOutput {
|
|
2
|
+
data: number[];
|
|
3
|
+
dimensions: number;
|
|
4
|
+
}
|
|
5
|
+
export interface IEmbeddingService {
|
|
6
|
+
init(): Promise<void>;
|
|
7
|
+
embed(text: string): Promise<EmbeddingOutput>;
|
|
8
|
+
getDimensions(): number;
|
|
9
|
+
}
|
|
10
|
+
//# sourceMappingURL=types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/engine/embeddings/types.ts"],"names":[],"mappings":"AAAA,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,iBAAiB;IAChC,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9C,aAAa,IAAI,MAAM,CAAC;CACzB"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../../src/engine/embeddings/types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { IEmbeddingService, EmbeddingOutput } from './types.js';
|
|
2
|
+
export declare class VertexEmbeddingService implements IEmbeddingService {
|
|
3
|
+
private config;
|
|
4
|
+
private client;
|
|
5
|
+
private model;
|
|
6
|
+
constructor(config: any);
|
|
7
|
+
init(): Promise<void>;
|
|
8
|
+
embed(text: string): Promise<EmbeddingOutput>;
|
|
9
|
+
getDimensions(): number;
|
|
10
|
+
}
|
|
11
|
+
//# sourceMappingURL=vertex-service.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vertex-service.d.ts","sourceRoot":"","sources":["../../../src/engine/embeddings/vertex-service.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,iBAAiB,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAGhE,qBAAa,sBAAuB,YAAW,iBAAiB;IAC9D,OAAO,CAAC,MAAM,CAAM;IACpB,OAAO,CAAC,MAAM,CAAyB;IACvC,OAAO,CAAC,KAAK,CAAgC;gBAEjC,MAAM,EAAE,GAAG;IAIjB,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAkBrB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmBnD,aAAa,IAAI,MAAM;CAGxB"}
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { VertexAI } from '@google-cloud/vertexai';
|
|
2
|
+
export class VertexEmbeddingService {
|
|
3
|
+
config;
|
|
4
|
+
client = null;
|
|
5
|
+
model = null;
|
|
6
|
+
constructor(config) {
|
|
7
|
+
this.config = config;
|
|
8
|
+
}
|
|
9
|
+
async init() {
|
|
10
|
+
if (!this.config.projectId || !this.config.location) {
|
|
11
|
+
if (!this.config.projectId)
|
|
12
|
+
throw new Error("Vertex AI requires 'projectId' in config or env");
|
|
13
|
+
if (!this.config.location)
|
|
14
|
+
throw new Error("Vertex AI requires 'location' in config or env");
|
|
15
|
+
}
|
|
16
|
+
this.client = new VertexAI({
|
|
17
|
+
project: this.config.projectId,
|
|
18
|
+
location: this.config.location,
|
|
19
|
+
});
|
|
20
|
+
// 'gemini-embedding-001' or user config
|
|
21
|
+
const modelName = this.config.embeddingModel || 'gemini-embedding-001';
|
|
22
|
+
this.model = this.client.getGenerativeModel({ model: modelName });
|
|
23
|
+
console.log(`[Vertex] Embedding Service Initialized: ${modelName} (${this.config.projectId}/${this.config.location})`);
|
|
24
|
+
}
|
|
25
|
+
async embed(text) {
|
|
26
|
+
if (!this.model)
|
|
27
|
+
await this.init();
|
|
28
|
+
// Cast to any to bypass TS error if embedContent is missing in type definition but present in runtime
|
|
29
|
+
// or if version mismatch.
|
|
30
|
+
const result = await this.model.embedContent(text);
|
|
31
|
+
// handling response structure
|
|
32
|
+
const embedding = result.embedding;
|
|
33
|
+
if (!embedding || !embedding.values) {
|
|
34
|
+
throw new Error('Vertex AI returned empty embedding');
|
|
35
|
+
}
|
|
36
|
+
return {
|
|
37
|
+
data: embedding.values,
|
|
38
|
+
dimensions: embedding.values.length
|
|
39
|
+
};
|
|
40
|
+
}
|
|
41
|
+
getDimensions() {
|
|
42
|
+
return 768;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
//# sourceMappingURL=vertex-service.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vertex-service.js","sourceRoot":"","sources":["../../../src/engine/embeddings/vertex-service.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,QAAQ,EAAmB,MAAM,wBAAwB,CAAC;AAEnE,MAAM,OAAO,sBAAsB;IACzB,MAAM,CAAM;IACZ,MAAM,GAAoB,IAAI,CAAC;IAC/B,KAAK,GAA2B,IAAI,CAAC;IAE7C,YAAY,MAAW;QACrB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS;gBAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;YAC/F,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,gDAAgD,CAAC,CAAC;QACjG,CAAC;QAED,IAAI,CAAC,MAAM,GAAG,IAAI,QAAQ,CAAC;YACvB,OAAO,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS;YAC9B,QAAQ,EAAE,IAAI,CAAC,MAAM,CAAC,QAAQ;SACjC,CAAC,CAAC;QAEH,wCAAwC;QACxC,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,CAAC,cAAc,IAAI,sBAAsB,CAAC;QACvE,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC,EAAE,KAAK,EAAE,SAAS,EAAE,CAAC,CAAC;QAElE,OAAO,CAAC,GAAG,CAAC,2CAA2C,SAAS,KAAK,IAAI,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,QAAQ,GAAG,CAAC,CAAC;IACzH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,CAAC,IAAI,CAAC,KAAK;YAAE,MAAM,IAAI,CAAC,IAAI,EAAE,CAAC;QAEnC,sGAAsG;QACtG,0BAA0B;QAC1B,MAAM,MAAM,GAAG,MAAO,IAAI,CAAC,KAAa,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC;QAE5D,8BAA8B;QAC9B,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,CAAC;QACnC,IAAI,CAAC,SAAS,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC;YAClC,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;QAC1D,CAAC;QAED,OAAO;YACL,IAAI,EAAE,SAAS,CAAC,MAAM;YACtB,UAAU,EAAE,SAAS,CAAC,MAAM,CAAC,MAAM;SACpC,CAAC;IACJ,CAAC;IAED,aAAa;QACX,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { IVectorStore } from './vector-store-types.js';
|
|
2
|
+
export declare class Indexer {
|
|
3
|
+
private queue;
|
|
4
|
+
private chunker;
|
|
5
|
+
private vectorStore;
|
|
6
|
+
constructor(vectorStore: IVectorStore);
|
|
7
|
+
enqueue(filePath: string): Promise<any>;
|
|
8
|
+
enqueueDelete(filePath: string): Promise<any>;
|
|
9
|
+
private worker;
|
|
10
|
+
private handleDelete;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../src/engine/indexer.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAOvD,qBAAa,OAAO;IAClB,OAAO,CAAC,KAAK,CAA4B;IACzC,OAAO,CAAC,OAAO,CAAkB;IACjC,OAAO,CAAC,WAAW,CAAe;gBAEtB,WAAW,EAAE,YAAY;IAO/B,OAAO,CAAC,QAAQ,EAAE,MAAM;IAIxB,aAAa,CAAC,QAAQ,EAAE,MAAM;YAItB,MAAM;YAsDN,YAAY;CAI3B"}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import fastq from 'fastq';
|
|
2
|
+
import fs from 'fs-extra';
|
|
3
|
+
import { globalMutex } from './lock.js';
|
|
4
|
+
import { SemanticChunker } from './chunker.js';
|
|
5
|
+
export class Indexer {
|
|
6
|
+
queue;
|
|
7
|
+
chunker;
|
|
8
|
+
vectorStore;
|
|
9
|
+
constructor(vectorStore) {
|
|
10
|
+
this.vectorStore = vectorStore;
|
|
11
|
+
this.chunker = new SemanticChunker();
|
|
12
|
+
// Concurrency 1 (Serialized)
|
|
13
|
+
this.queue = fastq.promise(this, this.worker, 1);
|
|
14
|
+
}
|
|
15
|
+
async enqueue(filePath) {
|
|
16
|
+
return this.queue.push({ filePath, type: 'index' });
|
|
17
|
+
}
|
|
18
|
+
async enqueueDelete(filePath) {
|
|
19
|
+
return this.queue.push({ filePath, type: 'delete' });
|
|
20
|
+
}
|
|
21
|
+
async worker(job) {
|
|
22
|
+
const { filePath, type } = job;
|
|
23
|
+
// Acquire Global Lock (Wait for API /save)
|
|
24
|
+
const release = await globalMutex.acquire();
|
|
25
|
+
try {
|
|
26
|
+
if (type === 'delete') {
|
|
27
|
+
await this.handleDelete(filePath);
|
|
28
|
+
return;
|
|
29
|
+
}
|
|
30
|
+
console.log(`[Indexer] Processing ${filePath}`);
|
|
31
|
+
if (!await fs.pathExists(filePath)) {
|
|
32
|
+
console.log(`[Indexer] File no longer exists: ${filePath}`);
|
|
33
|
+
await this.handleDelete(filePath);
|
|
34
|
+
return;
|
|
35
|
+
}
|
|
36
|
+
const content = await fs.readFile(filePath, 'utf8');
|
|
37
|
+
// Determining language from extension
|
|
38
|
+
let lang = 'typescript'; // Default
|
|
39
|
+
if (filePath.endsWith('.js') || filePath.endsWith('.mjs'))
|
|
40
|
+
lang = 'javascript';
|
|
41
|
+
else if (filePath.endsWith('.tsx'))
|
|
42
|
+
lang = 'tsx';
|
|
43
|
+
const chunks = await this.chunker.chunk(content, lang);
|
|
44
|
+
const entries = chunks.map(c => ({
|
|
45
|
+
id: '', // Generated on insert
|
|
46
|
+
mode: 'private',
|
|
47
|
+
date: new Date().toISOString(),
|
|
48
|
+
text: c.text,
|
|
49
|
+
source: filePath,
|
|
50
|
+
metadata: {
|
|
51
|
+
...c.metadata
|
|
52
|
+
}
|
|
53
|
+
}));
|
|
54
|
+
// Delete old entries for this source before adding new ones
|
|
55
|
+
const deleted = await this.vectorStore.deleteBySource(filePath);
|
|
56
|
+
if (deleted > 0) {
|
|
57
|
+
console.log(`[Indexer] Removed ${deleted} old entries for ${filePath}`);
|
|
58
|
+
}
|
|
59
|
+
await this.vectorStore.add(entries);
|
|
60
|
+
console.log(`[Indexer] Indexed ${entries.length} chunks from ${filePath}`);
|
|
61
|
+
}
|
|
62
|
+
catch (error) {
|
|
63
|
+
console.error(`[Indexer] Failed to index ${filePath}:`, error);
|
|
64
|
+
}
|
|
65
|
+
finally {
|
|
66
|
+
release();
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async handleDelete(filePath) {
|
|
70
|
+
const deleted = await this.vectorStore.deleteBySource(filePath);
|
|
71
|
+
console.log(`[Indexer] Deleted ${deleted} entries for removed file: ${filePath}`);
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../src/engine/indexer.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,MAAM,OAAO,CAAC;AAE1B,OAAO,EAAE,MAAM,UAAU,CAAC;AAE1B,OAAO,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACxC,OAAO,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAQ/C,MAAM,OAAO,OAAO;IACV,KAAK,CAA4B;IACjC,OAAO,CAAkB;IACzB,WAAW,CAAe;IAElC,YAAY,WAAyB;QACnC,IAAI,CAAC,WAAW,GAAG,WAAW,CAAC;QAC/B,IAAI,CAAC,OAAO,GAAG,IAAI,eAAe,EAAE,CAAC;QACrC,6BAA6B;QAC7B,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,QAAgB;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,aAAa,CAAC,QAAgB;QAClC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,MAAM,CAAC,GAAa;QAChC,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,GAAG,CAAC;QAE/B,2CAA2C;QAC3C,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,OAAO,EAAE,CAAC;QAC5C,IAAI,CAAC;YACH,IAAI,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtB,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YAED,OAAO,CAAC,GAAG,CAAC,wBAAwB,QAAQ,EAAE,CAAC,CAAC;YAChD,IAAI,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACnC,OAAO,CAAC,GAAG,CAAC,oCAAoC,QAAQ,EAAE,CAAC,CAAC;gBAC5D,MAAM,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC;gBAClC,OAAO;YACT,CAAC;YAED,MAAM,OAAO,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;YAEpD,sCAAsC;YACtC,IAAI,IAAI,GAAwC,YAAY,CAAC,CAAC,UAAU;YACxE,IAAI,QAAQ,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,IAAI,GAAG,YAAY,CAAC;iBAC1E,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,IAAI,GAAG,KAAK,CAAC;YAEjD,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;YAEvD,MAAM,OAAO,GAAY,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;gBACxC,EAAE,EAAE,EAAE,EAAE,sBAAsB;gBAC9B,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;gBAC9B,IAAI,EAAE,CAAC,CAAC,IAAI;gBACZ,MAAM,EAAE,QAAQ;gBAChB,QAAQ,EAAE;oBACR,GAAG,CAAC,CAAC,QAAQ;iBACd;aACF,CAAC,CAAC,CAAC;YAEJ,4DAA4D;YAC5D,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;YAChE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,oBAAoB,QAAQ,EAAE,CAAC,CAAC;YAC1E,CAAC;YAED,MAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;YACpC,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,CAAC,MAAM,gBAAgB,QAAQ,EAAE,CAAC,CAAC;QAE7E,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,OAAO,CAAC,KAAK,CAAC,6BAA6B,QAAQ,GAAG,EAAE,KAAK,CAAC,CAAC;QACjE,CAAC;gBAAS,CAAC;YACT,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,QAAgB;QACzC,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,WAAW,CAAC,cAAc,CAAC,QAAQ,CAAC,CAAC;QAChE,OAAO,CAAC,GAAG,CAAC,qBAAqB,OAAO,8BAA8B,QAAQ,EAAE,CAAC,CAAC;IACpF,CAAC;CACF"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YuiHub V1 Backend - Lock Utilities
|
|
3
|
+
* ReadWriteLock for LanceDB concurrency control
|
|
4
|
+
*/
|
|
5
|
+
import { RWLock } from 'async-rwlock';
|
|
6
|
+
export declare const rwLock: RWLock;
|
|
7
|
+
/**
|
|
8
|
+
* Execute function with write lock (exclusive)
|
|
9
|
+
* Use for: /save, /checkpoints, indexing
|
|
10
|
+
*/
|
|
11
|
+
export declare function withWriteLock<T>(fn: () => Promise<T>): Promise<T>;
|
|
12
|
+
/**
|
|
13
|
+
* Execute function with read lock (shared)
|
|
14
|
+
* Use for: /search, /export/context
|
|
15
|
+
*/
|
|
16
|
+
export declare function withReadLock<T>(fn: () => Promise<T>): Promise<T>;
|
|
17
|
+
/**
|
|
18
|
+
* Execute function with exponential backoff retry
|
|
19
|
+
*/
|
|
20
|
+
export declare function withRetry<T>(fn: () => Promise<T>, maxRetries?: number, baseDelayMs?: number): Promise<T>;
|
|
21
|
+
import { Mutex } from 'async-mutex';
|
|
22
|
+
/** @deprecated Use withWriteLock instead */
|
|
23
|
+
export declare const globalMutex: Mutex;
|
|
24
|
+
//# sourceMappingURL=lock.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lock.d.ts","sourceRoot":"","sources":["../../src/engine/lock.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,eAAO,MAAM,MAAM,QAAe,CAAC;AAEnC;;;GAGG;AACH,wBAAsB,aAAa,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAOvE;AAED;;;GAGG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAOtE;AAED;;GAEG;AACH,wBAAsB,SAAS,CAAC,CAAC,EAC/B,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EACpB,UAAU,SAAI,EACd,WAAW,SAAM,GAChB,OAAO,CAAC,CAAC,CAAC,CAiBZ;AAGD,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,4CAA4C;AAC5C,eAAO,MAAM,WAAW,OAAc,CAAC"}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* YuiHub V1 Backend - Lock Utilities
|
|
3
|
+
* ReadWriteLock for LanceDB concurrency control
|
|
4
|
+
*/
|
|
5
|
+
import { RWLock } from 'async-rwlock';
|
|
6
|
+
// ReadWriteLock: Multiple readers OR single writer
|
|
7
|
+
export const rwLock = new RWLock();
|
|
8
|
+
/**
|
|
9
|
+
* Execute function with write lock (exclusive)
|
|
10
|
+
* Use for: /save, /checkpoints, indexing
|
|
11
|
+
*/
|
|
12
|
+
export async function withWriteLock(fn) {
|
|
13
|
+
await rwLock.writeLock();
|
|
14
|
+
try {
|
|
15
|
+
return await fn();
|
|
16
|
+
}
|
|
17
|
+
finally {
|
|
18
|
+
rwLock.unlock();
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Execute function with read lock (shared)
|
|
23
|
+
* Use for: /search, /export/context
|
|
24
|
+
*/
|
|
25
|
+
export async function withReadLock(fn) {
|
|
26
|
+
await rwLock.readLock();
|
|
27
|
+
try {
|
|
28
|
+
return await fn();
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
rwLock.unlock();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Execute function with exponential backoff retry
|
|
36
|
+
*/
|
|
37
|
+
export async function withRetry(fn, maxRetries = 3, baseDelayMs = 100) {
|
|
38
|
+
let lastError;
|
|
39
|
+
for (let attempt = 0; attempt < maxRetries; attempt++) {
|
|
40
|
+
try {
|
|
41
|
+
return await fn();
|
|
42
|
+
}
|
|
43
|
+
catch (error) {
|
|
44
|
+
lastError = error;
|
|
45
|
+
if (attempt < maxRetries - 1) {
|
|
46
|
+
const delay = baseDelayMs * Math.pow(2, attempt);
|
|
47
|
+
console.warn(`[Lock] Retry ${attempt + 1}/${maxRetries} after ${delay}ms: ${lastError.message}`);
|
|
48
|
+
await new Promise(resolve => setTimeout(resolve, delay));
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
throw lastError;
|
|
53
|
+
}
|
|
54
|
+
// Legacy export for backward compatibility (deprecated)
|
|
55
|
+
import { Mutex } from 'async-mutex';
|
|
56
|
+
/** @deprecated Use withWriteLock instead */
|
|
57
|
+
export const globalMutex = new Mutex();
|
|
58
|
+
//# sourceMappingURL=lock.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lock.js","sourceRoot":"","sources":["../../src/engine/lock.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,mDAAmD;AACnD,MAAM,CAAC,MAAM,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;AAEnC;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CAAI,EAAoB;IACzD,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAI,EAAoB;IACxD,MAAM,MAAM,CAAC,QAAQ,EAAE,CAAC;IACxB,IAAI,CAAC;QACH,OAAO,MAAM,EAAE,EAAE,CAAC;IACpB,CAAC;YAAS,CAAC;QACT,MAAM,CAAC,MAAM,EAAE,CAAC;IAClB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,EAAoB,EACpB,UAAU,GAAG,CAAC,EACd,WAAW,GAAG,GAAG;IAEjB,IAAI,SAA4B,CAAC;IAEjC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;QACtD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAc,CAAC;YAC3B,IAAI,OAAO,GAAG,UAAU,GAAG,CAAC,EAAE,CAAC;gBAC7B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC;gBACjD,OAAO,CAAC,IAAI,CAAC,gBAAgB,OAAO,GAAG,CAAC,IAAI,UAAU,UAAU,KAAK,OAAO,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC;gBACjG,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC3D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,CAAC;AAClB,CAAC;AAED,wDAAwD;AACxD,OAAO,EAAE,KAAK,EAAE,MAAM,aAAa,CAAC;AACpC,4CAA4C;AAC5C,MAAM,CAAC,MAAM,WAAW,GAAG,IAAI,KAAK,EAAE,CAAC"}
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
import { Entry } from '@yuihub/core';
|
|
2
|
+
export interface LanceEntry {
|
|
3
|
+
id: string;
|
|
4
|
+
vector: number[];
|
|
5
|
+
text: string;
|
|
6
|
+
mode: string;
|
|
7
|
+
tags: string;
|
|
8
|
+
session_id: string;
|
|
9
|
+
source: string;
|
|
10
|
+
date: string;
|
|
11
|
+
metadata: string;
|
|
12
|
+
}
|
|
13
|
+
/**
|
|
14
|
+
* Convert Core Entry to LanceDB Entry (without vector)
|
|
15
|
+
* Vector must be added separately.
|
|
16
|
+
* Tags are serialized to JSON string.
|
|
17
|
+
*/
|
|
18
|
+
export declare function toLanceEntryBase(entry: Entry): Omit<LanceEntry, 'vector'>;
|
|
19
|
+
//# sourceMappingURL=schema.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.d.ts","sourceRoot":"","sources":["../../src/engine/schema.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAIrC,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,KAAK,GAAG,IAAI,CAAC,UAAU,EAAE,QAAQ,CAAC,CAWzE"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Convert Core Entry to LanceDB Entry (without vector)
|
|
3
|
+
* Vector must be added separately.
|
|
4
|
+
* Tags are serialized to JSON string.
|
|
5
|
+
*/
|
|
6
|
+
export function toLanceEntryBase(entry) {
|
|
7
|
+
return {
|
|
8
|
+
id: entry.id,
|
|
9
|
+
text: entry.text,
|
|
10
|
+
mode: entry.mode,
|
|
11
|
+
tags: JSON.stringify(entry.tags || []), // Serialize to avoid Arrow type inference issues
|
|
12
|
+
session_id: entry.session_id || '',
|
|
13
|
+
source: entry.source || '',
|
|
14
|
+
date: entry.date,
|
|
15
|
+
metadata: JSON.stringify(entry.metadata || {})
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
//# sourceMappingURL=schema.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"schema.js","sourceRoot":"","sources":["../../src/engine/schema.ts"],"names":[],"mappings":"AAgBA;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAAC,KAAY;IAC3C,OAAO;QACL,EAAE,EAAE,KAAK,CAAC,EAAE;QACZ,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,iDAAiD;QACzF,UAAU,EAAE,KAAK,CAAC,UAAU,IAAI,EAAE;QAClC,MAAM,EAAE,KAAK,CAAC,MAAM,IAAI,EAAE;QAC1B,IAAI,EAAE,KAAK,CAAC,IAAI;QAChB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,QAAQ,IAAI,EAAE,CAAC;KAC/C,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
import { Entry } from '@yuihub/core';
|
|
2
|
+
export interface SearchResult {
|
|
3
|
+
id: string;
|
|
4
|
+
text: string;
|
|
5
|
+
score: number;
|
|
6
|
+
mode: string;
|
|
7
|
+
tags: string;
|
|
8
|
+
session_id: string;
|
|
9
|
+
source: string;
|
|
10
|
+
date: string;
|
|
11
|
+
metadata: string;
|
|
12
|
+
_source_store?: string;
|
|
13
|
+
}
|
|
14
|
+
export interface IVectorStore {
|
|
15
|
+
init(): Promise<void>;
|
|
16
|
+
add(entries: Entry[]): Promise<void>;
|
|
17
|
+
search(query: string, limit?: number, filter?: {
|
|
18
|
+
tag?: string;
|
|
19
|
+
session?: string;
|
|
20
|
+
}): Promise<SearchResult[]>;
|
|
21
|
+
isEmpty(): Promise<boolean>;
|
|
22
|
+
deleteBySource(source: string): Promise<number>;
|
|
23
|
+
}
|
|
24
|
+
//# sourceMappingURL=vector-store-types.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store-types.d.ts","sourceRoot":"","sources":["../../src/engine/vector-store-types.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAErC,MAAM,WAAW,YAAY;IAC3B,EAAE,EAAE,MAAM,CAAC;IACX,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;IACb,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAED,MAAM,WAAW,YAAY;IAC3B,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACtB,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACrC,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC,CAAC;IAC5G,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC,CAAC;IAC5B,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACjD"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store-types.js","sourceRoot":"","sources":["../../src/engine/vector-store-types.ts"],"names":[],"mappings":""}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { Entry } from '@yuihub/core';
|
|
2
|
+
import { IEmbeddingService } from './embeddings/types.js';
|
|
3
|
+
import { IVectorStore, SearchResult } from './vector-store-types.js';
|
|
4
|
+
export declare class LanceVectorStore implements IVectorStore {
|
|
5
|
+
private db;
|
|
6
|
+
private table;
|
|
7
|
+
private embedder;
|
|
8
|
+
private dbPath;
|
|
9
|
+
private tableName;
|
|
10
|
+
readonly name: string;
|
|
11
|
+
constructor(basePath: string, embedder: IEmbeddingService, name?: string, tableName?: string);
|
|
12
|
+
init(): Promise<void>;
|
|
13
|
+
add(entries: Entry[]): Promise<void>;
|
|
14
|
+
search(query: string, limit?: number, filter?: {
|
|
15
|
+
tag?: string;
|
|
16
|
+
session?: string;
|
|
17
|
+
}): Promise<SearchResult[]>;
|
|
18
|
+
isEmpty(): Promise<boolean>;
|
|
19
|
+
deleteBySource(source: string): Promise<number>;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=vector-store.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"vector-store.d.ts","sourceRoot":"","sources":["../../src/engine/vector-store.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,KAAK,EAAE,MAAM,cAAc,CAAC;AAErC,OAAO,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAC1D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,yBAAyB,CAAC;AAErE,qBAAa,gBAAiB,YAAW,YAAY;IACnD,OAAO,CAAC,EAAE,CAAmC;IAC7C,OAAO,CAAC,KAAK,CAA8B;IAC3C,OAAO,CAAC,QAAQ,CAAoB;IACpC,OAAO,CAAC,MAAM,CAAS;IACvB,OAAO,CAAC,SAAS,CAAS;IAC1B,SAAgB,IAAI,EAAE,MAAM,CAAC;gBAEjB,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,iBAAiB,EAAE,IAAI,GAAE,MAAyB,EAAE,SAAS,CAAC,EAAE,MAAM;IAOxG,IAAI;IAcJ,GAAG,CAAC,OAAO,EAAE,KAAK,EAAE;IA0BpB,MAAM,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,GAAE,MAAW,EAAE,MAAM,CAAC,EAAE;QAAE,GAAG,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;IA6B/G,OAAO,IAAI,OAAO,CAAC,OAAO,CAAC;IAU3B,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,CAAC;CAetD"}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import * as lancedb from '@lancedb/lancedb';
|
|
2
|
+
import path from 'path';
|
|
3
|
+
import fs from 'fs-extra';
|
|
4
|
+
import { toLanceEntryBase } from './schema.js';
|
|
5
|
+
export class LanceVectorStore {
|
|
6
|
+
db = null;
|
|
7
|
+
table = null;
|
|
8
|
+
embedder;
|
|
9
|
+
dbPath;
|
|
10
|
+
tableName;
|
|
11
|
+
name; // Identifier for RRF (e.g. 'local', 'vertex')
|
|
12
|
+
constructor(basePath, embedder, name = 'yuihub_entries', tableName) {
|
|
13
|
+
this.dbPath = path.join(basePath, 'data/lancedb');
|
|
14
|
+
this.embedder = embedder;
|
|
15
|
+
this.name = name;
|
|
16
|
+
this.tableName = tableName || `entries_${name}`; // e.g. entries_local, entries_vertex
|
|
17
|
+
}
|
|
18
|
+
async init() {
|
|
19
|
+
await fs.ensureDir(this.dbPath);
|
|
20
|
+
this.db = await lancedb.connect(this.dbPath);
|
|
21
|
+
// Initialize Embedder
|
|
22
|
+
await this.embedder.init();
|
|
23
|
+
// Check if table exists
|
|
24
|
+
const tableNames = await this.db.tableNames();
|
|
25
|
+
if (tableNames.includes(this.tableName)) {
|
|
26
|
+
this.table = await this.db.openTable(this.tableName);
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
async add(entries) {
|
|
30
|
+
if (!this.db)
|
|
31
|
+
throw new Error('VectorStore not initialized');
|
|
32
|
+
const data = [];
|
|
33
|
+
for (const entry of entries) {
|
|
34
|
+
const output = await this.embedder.embed(entry.text);
|
|
35
|
+
// Serialize tags array to JSON string to avoid Arrow type inference issues
|
|
36
|
+
const baseEntry = toLanceEntryBase(entry);
|
|
37
|
+
data.push({
|
|
38
|
+
...baseEntry,
|
|
39
|
+
vector: output.data
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
if (data.length === 0)
|
|
43
|
+
return;
|
|
44
|
+
if (!this.table) {
|
|
45
|
+
// @ts-ignore: LanceDB Type mismatch with strict mode
|
|
46
|
+
this.table = await this.db.createTable(this.tableName, data);
|
|
47
|
+
}
|
|
48
|
+
else {
|
|
49
|
+
// @ts-ignore: LanceDB Type mismatch with strict mode
|
|
50
|
+
await this.table.add(data);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
async search(query, limit = 10, filter) {
|
|
54
|
+
if (!this.table)
|
|
55
|
+
return [];
|
|
56
|
+
const output = await this.embedder.embed(query);
|
|
57
|
+
const vector = output.data;
|
|
58
|
+
let builder = this.table.search(vector).limit(limit);
|
|
59
|
+
const whereClauses = [];
|
|
60
|
+
if (filter?.session) {
|
|
61
|
+
whereClauses.push(`session_id = '${filter.session}'`);
|
|
62
|
+
}
|
|
63
|
+
if (whereClauses.length > 0) {
|
|
64
|
+
builder = builder.where(whereClauses.join(' AND '));
|
|
65
|
+
}
|
|
66
|
+
const rows = await builder.toArray();
|
|
67
|
+
return rows.map((r) => ({
|
|
68
|
+
...r,
|
|
69
|
+
score: r._distance, // LanceDB uses distance (lower is better usually for L2, cosine dist?)
|
|
70
|
+
// Note: OpenAI/MiniLM usually use Cosine Similarity or Distance.
|
|
71
|
+
// LanceDB 'cosine' distance = 1 - similarity.
|
|
72
|
+
// RRF needs consistent ranking. Distance is fine for ascending rank.
|
|
73
|
+
_source_store: this.name
|
|
74
|
+
}));
|
|
75
|
+
}
|
|
76
|
+
async isEmpty() {
|
|
77
|
+
if (!this.table)
|
|
78
|
+
return true;
|
|
79
|
+
try {
|
|
80
|
+
const count = await this.table.countRows();
|
|
81
|
+
return count === 0;
|
|
82
|
+
}
|
|
83
|
+
catch {
|
|
84
|
+
return true;
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
async deleteBySource(source) {
|
|
88
|
+
if (!this.table)
|
|
89
|
+
return 0;
|
|
90
|
+
try {
|
|
91
|
+
// LanceDB delete API - returns void, we count before/after
|
|
92
|
+
const countBefore = await this.table.countRows();
|
|
93
|
+
// Escape single quotes in source path
|
|
94
|
+
const escapedSource = source.replace(/'/g, "''");
|
|
95
|
+
await this.table.delete(`source = '${escapedSource}'`);
|
|
96
|
+
const countAfter = await this.table.countRows();
|
|
97
|
+
return countBefore - countAfter;
|
|
98
|
+
}
|
|
99
|
+
catch (e) {
|
|
100
|
+
console.error(`[LanceVectorStore] deleteBySource failed for ${source}:`, e);
|
|
101
|
+
return 0;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
//# sourceMappingURL=vector-store.js.map
|