@venturekit-pro/ai 0.0.0-dev.20260323005827
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +19 -0
- package/README.md +117 -0
- package/dist/agents/agent.d.ts +22 -0
- package/dist/agents/agent.d.ts.map +1 -0
- package/dist/agents/agent.js +138 -0
- package/dist/agents/agent.js.map +1 -0
- package/dist/agents/config.d.ts +13 -0
- package/dist/agents/config.d.ts.map +1 -0
- package/dist/agents/config.js +24 -0
- package/dist/agents/config.js.map +1 -0
- package/dist/agents/index.d.ts +10 -0
- package/dist/agents/index.d.ts.map +1 -0
- package/dist/agents/index.js +9 -0
- package/dist/agents/index.js.map +1 -0
- package/dist/agents/tools.d.ts +14 -0
- package/dist/agents/tools.d.ts.map +1 -0
- package/dist/agents/tools.js +15 -0
- package/dist/agents/tools.js.map +1 -0
- package/dist/context/context.d.ts +118 -0
- package/dist/context/context.d.ts.map +1 -0
- package/dist/context/context.js +132 -0
- package/dist/context/context.js.map +1 -0
- package/dist/embeddings/config.d.ts +13 -0
- package/dist/embeddings/config.d.ts.map +1 -0
- package/dist/embeddings/config.js +25 -0
- package/dist/embeddings/config.js.map +1 -0
- package/dist/embeddings/embedder.d.ts +20 -0
- package/dist/embeddings/embedder.d.ts.map +1 -0
- package/dist/embeddings/embedder.js +100 -0
- package/dist/embeddings/embedder.js.map +1 -0
- package/dist/embeddings/index.d.ts +9 -0
- package/dist/embeddings/index.d.ts.map +1 -0
- package/dist/embeddings/index.js +8 -0
- package/dist/embeddings/index.js.map +1 -0
- package/dist/index.d.ts +18 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +19 -0
- package/dist/index.js.map +1 -0
- package/dist/rag/chunker.d.ts +11 -0
- package/dist/rag/chunker.d.ts.map +1 -0
- package/dist/rag/chunker.js +109 -0
- package/dist/rag/chunker.js.map +1 -0
- package/dist/rag/config.d.ts +13 -0
- package/dist/rag/config.d.ts.map +1 -0
- package/dist/rag/config.js +35 -0
- package/dist/rag/config.js.map +1 -0
- package/dist/rag/index.d.ts +10 -0
- package/dist/rag/index.d.ts.map +1 -0
- package/dist/rag/index.js +9 -0
- package/dist/rag/index.js.map +1 -0
- package/dist/rag/pipeline.d.ts +26 -0
- package/dist/rag/pipeline.d.ts.map +1 -0
- package/dist/rag/pipeline.js +79 -0
- package/dist/rag/pipeline.js.map +1 -0
- package/dist/types/agents.d.ts +118 -0
- package/dist/types/agents.d.ts.map +1 -0
- package/dist/types/agents.js +5 -0
- package/dist/types/agents.js.map +1 -0
- package/dist/types/embeddings.d.ts +69 -0
- package/dist/types/embeddings.d.ts.map +1 -0
- package/dist/types/embeddings.js +5 -0
- package/dist/types/embeddings.js.map +1 -0
- package/dist/types/index.d.ts +8 -0
- package/dist/types/index.d.ts.map +1 -0
- package/dist/types/index.js +8 -0
- package/dist/types/index.js.map +1 -0
- package/dist/types/rag.d.ts +92 -0
- package/dist/types/rag.d.ts.map +1 -0
- package/dist/types/rag.js +5 -0
- package/dist/types/rag.js.map +1 -0
- package/dist/types/vectors.d.ts +98 -0
- package/dist/types/vectors.d.ts.map +1 -0
- package/dist/types/vectors.js +5 -0
- package/dist/types/vectors.js.map +1 -0
- package/dist/vectors/config.d.ts +13 -0
- package/dist/vectors/config.d.ts.map +1 -0
- package/dist/vectors/config.js +24 -0
- package/dist/vectors/config.js.map +1 -0
- package/dist/vectors/index.d.ts +9 -0
- package/dist/vectors/index.d.ts.map +1 -0
- package/dist/vectors/index.js +8 -0
- package/dist/vectors/index.js.map +1 -0
- package/dist/vectors/store.d.ts +24 -0
- package/dist/vectors/store.d.ts.map +1 -0
- package/dist/vectors/store.js +156 -0
- package/dist/vectors/store.js.map +1 -0
- package/package.json +63 -0
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* VentureKit AI Context Integration
|
|
3
|
+
*
|
|
4
|
+
* Provides tenant-aware, permission-aware AI capabilities.
|
|
5
|
+
* Wraps agents and RAG pipelines with VentureKit context
|
|
6
|
+
* so that all AI operations are scoped to the current tenant
|
|
7
|
+
* and respect user permissions.
|
|
8
|
+
*/
|
|
9
|
+
/**
|
|
10
|
+
* Build a system prompt suffix with VentureKit context
|
|
11
|
+
*/
|
|
12
|
+
export function buildContextPrompt(context) {
|
|
13
|
+
const parts = [];
|
|
14
|
+
parts.push(`\n[Context: tenant=${context.tenantId}`);
|
|
15
|
+
if (context.userId)
|
|
16
|
+
parts.push(`, user=${context.userId}`);
|
|
17
|
+
if (context.permissions?.length)
|
|
18
|
+
parts.push(`, permissions=${context.permissions.join(',')}`);
|
|
19
|
+
if (context.metadata) {
|
|
20
|
+
for (const [key, value] of Object.entries(context.metadata)) {
|
|
21
|
+
parts.push(`, ${key}=${String(value)}`);
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
parts.push(']');
|
|
25
|
+
return parts.join('');
|
|
26
|
+
}
|
|
27
|
+
/**
|
|
28
|
+
* Create a tenant-scoped metadata filter for RAG queries
|
|
29
|
+
*/
|
|
30
|
+
export function tenantFilter(tenantId, additionalFilters) {
|
|
31
|
+
return {
|
|
32
|
+
tenantId,
|
|
33
|
+
...additionalFilters,
|
|
34
|
+
};
|
|
35
|
+
}
|
|
36
|
+
/**
|
|
37
|
+
* Create a context-aware agent that scopes all operations to a tenant
|
|
38
|
+
*/
|
|
39
|
+
export function createContextAwareAgent(context, options) {
|
|
40
|
+
const { agentConfig, createAgent: agentFactory, checkPermission, requiredPermission, injectContext = true, } = options;
|
|
41
|
+
// Inject context into system prompt if enabled
|
|
42
|
+
const config = { ...agentConfig };
|
|
43
|
+
if (injectContext) {
|
|
44
|
+
const contextSuffix = buildContextPrompt(context);
|
|
45
|
+
config.systemPrompt = (config.systemPrompt || '') + contextSuffix;
|
|
46
|
+
}
|
|
47
|
+
const agent = agentFactory(config);
|
|
48
|
+
return {
|
|
49
|
+
/**
|
|
50
|
+
* Run with permission check
|
|
51
|
+
*/
|
|
52
|
+
async run(prompt) {
|
|
53
|
+
if (checkPermission && requiredPermission) {
|
|
54
|
+
const allowed = await checkPermission(context, requiredPermission, 'agent');
|
|
55
|
+
if (!allowed) {
|
|
56
|
+
throw new Error(`Permission denied: user '${context.userId}' lacks '${requiredPermission}' for tenant '${context.tenantId}'`);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return agent.run(prompt);
|
|
60
|
+
},
|
|
61
|
+
/**
|
|
62
|
+
* Chat with permission check
|
|
63
|
+
*/
|
|
64
|
+
async chat(messages) {
|
|
65
|
+
if (checkPermission && requiredPermission) {
|
|
66
|
+
const allowed = await checkPermission(context, requiredPermission, 'agent');
|
|
67
|
+
if (!allowed) {
|
|
68
|
+
throw new Error(`Permission denied: user '${context.userId}' lacks '${requiredPermission}' for tenant '${context.tenantId}'`);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return agent.chat(messages);
|
|
72
|
+
},
|
|
73
|
+
/**
|
|
74
|
+
* Add a tool to the underlying agent
|
|
75
|
+
*/
|
|
76
|
+
addTool(tool) {
|
|
77
|
+
agent.addTool(tool);
|
|
78
|
+
},
|
|
79
|
+
/** The VentureKit context */
|
|
80
|
+
context,
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Create a context-aware RAG pipeline that scopes queries to a tenant
|
|
85
|
+
*/
|
|
86
|
+
export function createContextAwareRag(context, options) {
|
|
87
|
+
const { createPipeline, ragConfig, checkPermission, requiredPermission, } = options;
|
|
88
|
+
const pipeline = createPipeline(ragConfig);
|
|
89
|
+
return {
|
|
90
|
+
/**
|
|
91
|
+
* Index documents with tenant metadata automatically injected
|
|
92
|
+
*/
|
|
93
|
+
async index(documents) {
|
|
94
|
+
const scopedDocs = documents.map(doc => ({
|
|
95
|
+
...doc,
|
|
96
|
+
id: `${context.tenantId}:${doc.id}`,
|
|
97
|
+
metadata: {
|
|
98
|
+
...doc.metadata,
|
|
99
|
+
tenantId: context.tenantId,
|
|
100
|
+
indexedBy: context.userId,
|
|
101
|
+
indexedAt: new Date().toISOString(),
|
|
102
|
+
},
|
|
103
|
+
}));
|
|
104
|
+
return pipeline.index(scopedDocs);
|
|
105
|
+
},
|
|
106
|
+
/**
|
|
107
|
+
* Retrieve with automatic tenant filter
|
|
108
|
+
*/
|
|
109
|
+
async retrieve(query, options) {
|
|
110
|
+
if (checkPermission && requiredPermission) {
|
|
111
|
+
const allowed = await checkPermission(context, requiredPermission, 'rag');
|
|
112
|
+
if (!allowed) {
|
|
113
|
+
throw new Error(`Permission denied: user '${context.userId}' lacks '${requiredPermission}' for tenant '${context.tenantId}'`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return pipeline.retrieve(query, {
|
|
117
|
+
...options,
|
|
118
|
+
filter: tenantFilter(context.tenantId, options?.filter),
|
|
119
|
+
});
|
|
120
|
+
},
|
|
121
|
+
/**
|
|
122
|
+
* Delete documents (scoped to tenant)
|
|
123
|
+
*/
|
|
124
|
+
async delete(ids) {
|
|
125
|
+
const scopedIds = ids.map(id => `${context.tenantId}:${id}`);
|
|
126
|
+
return pipeline.delete(scopedIds);
|
|
127
|
+
},
|
|
128
|
+
/** The VentureKit context */
|
|
129
|
+
context,
|
|
130
|
+
};
|
|
131
|
+
}
|
|
132
|
+
//# sourceMappingURL=context.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"context.js","sourceRoot":"","sources":["../../src/context/context.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAkEH;;GAEG;AACH,MAAM,UAAU,kBAAkB,CAAC,OAA4B;IAC7D,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,sBAAsB,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;IACrD,IAAI,OAAO,CAAC,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,UAAU,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAC3D,IAAI,OAAO,CAAC,WAAW,EAAE,MAAM;QAAE,KAAK,CAAC,IAAI,CAAC,iBAAiB,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAC9F,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC5D,KAAK,CAAC,IAAI,CAAC,KAAK,GAAG,IAAI,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;QAC1C,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IAChB,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,YAAY,CAC1B,QAAgB,EAChB,iBAA2C;IAE3C,OAAO;QACL,QAAQ;QACR,GAAG,iBAAiB;KACrB,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,uBAAuB,CACrC,OAA4B,EAC5B,OAAiC;IAEjC,MAAM,EACJ,WAAW,EACX,WAAW,EAAE,YAAY,EACzB,eAAe,EACf,kBAAkB,EAClB,aAAa,GAAG,IAAI,GACrB,GAAG,OAAO,CAAC;IAEZ,+CAA+C;IAC/C,MAAM,MAAM,GAAG,EAAE,GAAG,WAAW,EAAE,CAAC;IAClC,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,GAAG,kBAAkB,CAAC,OAAO,CAAC,CAAC;QAClD,MAAM,CAAC,YAAY,GAAG,CAAC,MAAM,CAAC,YAAY,IAAI,EAAE,CAAC,GAAG,aAAa,CAAC;IACpE,CAAC;IAED,MAAM,KAAK,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC;IAEnC,OAAO;QACL;;WAEG;QACH,KAAK,CAAC,GAAG,CAAC,MAAc;YACtB,IAAI,eAAe,IAAI,kBAAkB,EAAE,CAAC;gBAC1C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;gBAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,MAAM,YAAY,kBAAkB,iBAAiB,OAAO,CAAC,QAAQ,GAAG,CAC7G,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC3B,CAAC;QAED;;WAEG;QACH,KAAK,CAAC,IAAI,CAAC,QAAuB;YAChC,IAAI,eAAe,IAAI,kBAAkB,EAAE,CAAC;gBAC1C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,kBAAkB,EAAE,OAAO,CAAC,CAAC;gBAC5E,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,MAAM,YAAY,kBAAkB,iBAAiB,OAAO,CAAC,QAAQ,GAAG,CAC7G,CAAC;gBACJ,CAAC;YACH,CAAC;YACD,OAAO,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC9B,CAAC;QAED;;WAEG;QACH,OAAO,CAAC,IAAU;YAChB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC;QAED,6BAA6B;QAC7B,OAAO;KACR,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,qBAAqB,CACnC,OAA4B,EAC5B,OAA+B;IAE/B,MAAM,EACJ,cAAc,EACd,SAAS,EACT,eAAe,EACf,kBAAkB,GACnB,GAAG,OAAO,CAAC;IAEZ,MAAM,QAAQ,GAAG,cAAc,CAAC,SAAS,CAAC,CAAC;IAE3C,OAAO;QACL;;WAEG;QACH,KAAK,CAAC,KAAK,CACT,SAAqF;YAErF,MAAM,UAAU,GAAG,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;gBACvC,GAAG,GAAG;gBACN,EAAE,EAAE,GAAG,OAAO,CAAC,QAAQ,IAAI,GAAG,CAAC,EAAE,EAAE;gBACnC,QAAQ,EAAE;oBACR,GAAG,GAAG,CAAC,QAAQ;oBACf,QAAQ,EAAE,OAAO,CAAC,QAAQ;oBAC1B,SAAS,EAAE,OAAO,CAAC,MAAM;oBACzB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;iBACpC;aACF,CAAC,CAAC,CAAC;YACJ,OAAO,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACpC,CAAC;QAED;;WAEG;QACH,KAAK,CAAC,QAAQ,CACZ,KAAa,EACb,OAAyB;YAEzB,IAAI,eAAe,IAAI,kBAAkB,EAAE,CAAC;gBAC1C,MAAM,OAAO,GAAG,MAAM,eAAe,CAAC,OAAO,EAAE,kBAAkB,EAAE,KAAK,CAAC,CAAC;gBAC1E,IAAI,CAAC,OAAO,EAAE,CAAC;oBACb,MAAM,IAAI,KAAK,CACb,4BAA4B,OAAO,CAAC,MAAM,YAAY,kBAAkB,iBAAiB,OAAO,CAAC,QAAQ,GAAG,CAC7G,CAAC;gBACJ,CAAC;YACH,CAAC;YAED,OAAO,QAAQ,CAAC,QAAQ,CAAC,KAAK,EAAE;gBAC9B,GAAG,OAAO;gBACV,MAAM,EAAE,YAAY,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,EAAE,MAAM,CAAC;aACxD,CAAC,CAAC;QACL,CAAC;QAED;;WAEG;QACH,KAAK,CAAC,MAAM,CAAC,GAAa;YACxB,MAAM,SAAS,GAAG,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,QAAQ,IAAI,EAAE,EAAE,CAAC,CAAC;YAC7D,OAAO,QAAQ,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACpC,CAAC;QAED,6BAA6B;QAC7B,OAAO;KACR,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Configuration
|
|
3
|
+
*/
|
|
4
|
+
import type { EmbeddingConfig, EmbeddingConfigInput } from '../types/index.js';
|
|
5
|
+
/**
|
|
6
|
+
* Default embedding configuration (OpenAI)
|
|
7
|
+
*/
|
|
8
|
+
export declare const DEFAULT_EMBEDDING_CONFIG: EmbeddingConfig;
|
|
9
|
+
/**
|
|
10
|
+
* Create embedding configuration from input
|
|
11
|
+
*/
|
|
12
|
+
export declare function createEmbeddingConfig(input?: EmbeddingConfigInput): EmbeddingConfig;
|
|
13
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/embeddings/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAE/E;;GAEG;AACH,eAAO,MAAM,wBAAwB,EAAE,eAMtC,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,KAAK,CAAC,EAAE,oBAAoB,GAAG,eAAe,CAOnF"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding Configuration
|
|
3
|
+
*/
|
|
4
|
+
/**
|
|
5
|
+
* Default embedding configuration (OpenAI)
|
|
6
|
+
*/
|
|
7
|
+
export const DEFAULT_EMBEDDING_CONFIG = {
|
|
8
|
+
provider: 'openai',
|
|
9
|
+
model: 'text-embedding-3-small',
|
|
10
|
+
dimensions: 1536,
|
|
11
|
+
batchSize: 100,
|
|
12
|
+
maxRetries: 3,
|
|
13
|
+
};
|
|
14
|
+
/**
|
|
15
|
+
* Create embedding configuration from input
|
|
16
|
+
*/
|
|
17
|
+
export function createEmbeddingConfig(input) {
|
|
18
|
+
if (!input)
|
|
19
|
+
return DEFAULT_EMBEDDING_CONFIG;
|
|
20
|
+
return {
|
|
21
|
+
...DEFAULT_EMBEDDING_CONFIG,
|
|
22
|
+
...input,
|
|
23
|
+
};
|
|
24
|
+
}
|
|
25
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/embeddings/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAIH;;GAEG;AACH,MAAM,CAAC,MAAM,wBAAwB,GAAoB;IACvD,QAAQ,EAAE,QAAQ;IAClB,KAAK,EAAE,wBAAwB;IAC/B,UAAU,EAAE,IAAI;IAChB,SAAS,EAAE,GAAG;IACd,UAAU,EAAE,CAAC;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,qBAAqB,CAAC,KAA4B;IAChE,IAAI,CAAC,KAAK;QAAE,OAAO,wBAAwB,CAAC;IAE5C,OAAO;QACL,GAAG,wBAAwB;QAC3B,GAAG,KAAK;KACT,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder
|
|
3
|
+
*
|
|
4
|
+
* Unified interface for generating embeddings across providers.
|
|
5
|
+
*/
|
|
6
|
+
import type { EmbeddingConfig, EmbeddingResult, BatchEmbeddingResult } from '../types/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Embedder interface
|
|
9
|
+
*/
|
|
10
|
+
export interface Embedder {
|
|
11
|
+
/** Generate embedding for single text */
|
|
12
|
+
embed(text: string): Promise<EmbeddingResult>;
|
|
13
|
+
/** Generate embeddings for multiple texts */
|
|
14
|
+
embedBatch(texts: string[]): Promise<BatchEmbeddingResult>;
|
|
15
|
+
}
|
|
16
|
+
/**
|
|
17
|
+
* Create an embedder for the configured provider
|
|
18
|
+
*/
|
|
19
|
+
export declare function createEmbedder(config: EmbeddingConfig): Embedder;
|
|
20
|
+
//# sourceMappingURL=embedder.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/embeddings/embedder.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,eAAe,EAAE,eAAe,EAAE,oBAAoB,EAAE,MAAM,mBAAmB,CAAC;AAEhG;;GAEG;AACH,MAAM,WAAW,QAAQ;IACvB,yCAAyC;IACzC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAE9C,6CAA6C;IAC7C,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;CAC5D;AAED;;GAEG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,eAAe,GAAG,QAAQ,CAShE"}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedder
|
|
3
|
+
*
|
|
4
|
+
* Unified interface for generating embeddings across providers.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Create an embedder for the configured provider
|
|
8
|
+
*/
|
|
9
|
+
export function createEmbedder(config) {
|
|
10
|
+
switch (config.provider) {
|
|
11
|
+
case 'openai':
|
|
12
|
+
return createOpenAIEmbedder(config);
|
|
13
|
+
case 'bedrock':
|
|
14
|
+
return createBedrockEmbedder(config);
|
|
15
|
+
default:
|
|
16
|
+
throw new Error(`Unsupported embedding provider: ${config.provider}`);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/**
|
|
20
|
+
* OpenAI embedder implementation
|
|
21
|
+
*/
|
|
22
|
+
function createOpenAIEmbedder(config) {
|
|
23
|
+
return {
|
|
24
|
+
async embed(text) {
|
|
25
|
+
// Dynamic import to keep OpenAI optional
|
|
26
|
+
const { default: OpenAI } = await import('openai');
|
|
27
|
+
const client = new OpenAI({ apiKey: config.apiKey });
|
|
28
|
+
const response = await client.embeddings.create({
|
|
29
|
+
model: config.model,
|
|
30
|
+
input: text,
|
|
31
|
+
dimensions: config.dimensions,
|
|
32
|
+
});
|
|
33
|
+
return {
|
|
34
|
+
embedding: response.data[0].embedding,
|
|
35
|
+
tokensUsed: response.usage?.total_tokens,
|
|
36
|
+
model: config.model,
|
|
37
|
+
};
|
|
38
|
+
},
|
|
39
|
+
async embedBatch(texts) {
|
|
40
|
+
const { default: OpenAI } = await import('openai');
|
|
41
|
+
const client = new OpenAI({ apiKey: config.apiKey });
|
|
42
|
+
const batchSize = config.batchSize || 100;
|
|
43
|
+
const allEmbeddings = [];
|
|
44
|
+
let totalTokens = 0;
|
|
45
|
+
// Process in batches
|
|
46
|
+
for (let i = 0; i < texts.length; i += batchSize) {
|
|
47
|
+
const batch = texts.slice(i, i + batchSize);
|
|
48
|
+
const response = await client.embeddings.create({
|
|
49
|
+
model: config.model,
|
|
50
|
+
input: batch,
|
|
51
|
+
dimensions: config.dimensions,
|
|
52
|
+
});
|
|
53
|
+
// Sort by index to maintain order
|
|
54
|
+
const sorted = response.data.sort((a, b) => a.index - b.index);
|
|
55
|
+
allEmbeddings.push(...sorted.map(d => d.embedding));
|
|
56
|
+
totalTokens += response.usage?.total_tokens || 0;
|
|
57
|
+
}
|
|
58
|
+
return {
|
|
59
|
+
embeddings: allEmbeddings,
|
|
60
|
+
totalTokensUsed: totalTokens,
|
|
61
|
+
model: config.model,
|
|
62
|
+
};
|
|
63
|
+
},
|
|
64
|
+
};
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* AWS Bedrock embedder implementation
|
|
68
|
+
*/
|
|
69
|
+
function createBedrockEmbedder(config) {
|
|
70
|
+
return {
|
|
71
|
+
async embed(text) {
|
|
72
|
+
const { BedrockRuntimeClient, InvokeModelCommand } = await import('@aws-sdk/client-bedrock-runtime');
|
|
73
|
+
const client = new BedrockRuntimeClient({ region: config.region || 'us-east-1' });
|
|
74
|
+
const response = await client.send(new InvokeModelCommand({
|
|
75
|
+
modelId: config.model,
|
|
76
|
+
contentType: 'application/json',
|
|
77
|
+
accept: 'application/json',
|
|
78
|
+
body: JSON.stringify({ inputText: text }),
|
|
79
|
+
}));
|
|
80
|
+
const result = JSON.parse(new TextDecoder().decode(response.body));
|
|
81
|
+
return {
|
|
82
|
+
embedding: result.embedding,
|
|
83
|
+
model: config.model,
|
|
84
|
+
};
|
|
85
|
+
},
|
|
86
|
+
async embedBatch(texts) {
|
|
87
|
+
// Bedrock doesn't support batch, so we process sequentially
|
|
88
|
+
const embeddings = [];
|
|
89
|
+
for (const text of texts) {
|
|
90
|
+
const result = await this.embed(text);
|
|
91
|
+
embeddings.push(result.embedding);
|
|
92
|
+
}
|
|
93
|
+
return {
|
|
94
|
+
embeddings,
|
|
95
|
+
model: config.model,
|
|
96
|
+
};
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=embedder.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/embeddings/embedder.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAeH;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,MAAuB;IACpD,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,QAAQ;YACX,OAAO,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACtC,KAAK,SAAS;YACZ,OAAO,qBAAqB,CAAC,MAAM,CAAC,CAAC;QACvC;YACE,MAAM,IAAI,KAAK,CAAC,mCAAmC,MAAM,CAAC,QAAQ,EAAE,CAAC,CAAC;IAC1E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,oBAAoB,CAAC,MAAuB;IACnD,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,IAAY;YACtB,yCAAyC;YACzC,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAErD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;gBAC9C,KAAK,EAAE,MAAM,CAAC,KAAK;gBACnB,KAAK,EAAE,IAAI;gBACX,UAAU,EAAE,MAAM,CAAC,UAAU;aAC9B,CAAC,CAAC;YAEH,OAAO;gBACL,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS;gBACrC,UAAU,EAAE,QAAQ,CAAC,KAAK,EAAE,YAAY;gBACxC,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,KAAe;YAC9B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,MAAM,MAAM,CAAC,QAAQ,CAAC,CAAC;YACnD,MAAM,MAAM,GAAG,IAAI,MAAM,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;YAErD,MAAM,SAAS,GAAG,MAAM,CAAC,SAAS,IAAI,GAAG,CAAC;YAC1C,MAAM,aAAa,GAAe,EAAE,CAAC;YACrC,IAAI,WAAW,GAAG,CAAC,CAAC;YAEpB,qBAAqB;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,SAAS,EAAE,CAAC;gBACjD,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,SAAS,CAAC,CAAC;gBAC5C,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;oBAC9C,KAAK,EAAE,MAAM,CAAC,KAAK;oBACnB,KAAK,EAAE,KAAK;oBACZ,UAAU,EAAE,MAAM,CAAC,UAAU;iBAC9B,CAAC,CAAC;gBAEH,kCAAkC;gBAClC,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;gBAC/D,aAAa,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;gBACpD,WAAW,IAAI,QAAQ,CAAC,KAAK,EAAE,YAAY,IAAI,CAAC,CAAC;YACnD,CAAC;YAED,OAAO;gBACL,UAAU,EAAE,aAAa;gBACzB,eAAe,EAAE,WAAW;gBAC5B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,MAAuB;IACpD,OAAO;QACL,KAAK,CAAC,KAAK,CAAC,IAAY;YACtB,MAAM,EAAE,oBAAoB,EAAE,kBAAkB,EAAE,GAAG,MAAM,MAAM,CAAC,iCAAiC,CAAC,CAAC;YACrG,MAAM,MAAM,GAAG,IAAI,oBAAoB,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,IAAI,WAAW,EAAE,CAAC,CAAC;YAElF,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,IAAI,CAAC,IAAI,kBAAkB,CAAC;gBACxD,OAAO,EAAE,MAAM,CAAC,KAAK;gBACrB,WAAW,EAAE,kBAAkB;gBAC/B,MAAM,EAAE,kBAAkB;gBAC1B,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC;aAC1C,CAAC,CAAC,CAAC;YAEJ,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;YAEnE,OAAO;gBACL,SAAS,EAAE,MAAM,CAAC,SAAS;gBAC3B,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,KAAe;YAC9B,4DAA4D;YAC5D,MAAM,UAAU,GAAe,EAAE,CAAC;YAElC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;gBACtC,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YACpC,CAAC;YAED,OAAO;gBACL,UAAU;gBACV,KAAK,EAAE,MAAM,CAAC,KAAK;aACpB,CAAC;QACJ,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embeddings
|
|
3
|
+
*
|
|
4
|
+
* Utilities for generating text embeddings.
|
|
5
|
+
*/
|
|
6
|
+
export { createEmbeddingConfig, DEFAULT_EMBEDDING_CONFIG } from './config.js';
|
|
7
|
+
export { createEmbedder } from './embedder.js';
|
|
8
|
+
export type { Embedder } from './embedder.js';
|
|
9
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAC/C,YAAY,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/embeddings/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,qBAAqB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC"}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit-pro/ai
|
|
3
|
+
*
|
|
4
|
+
* AI utilities for VentureKit - embeddings, RAG, and agents.
|
|
5
|
+
* Documentation: https://venturekit.dev
|
|
6
|
+
*/
|
|
7
|
+
export * from './types/index.js';
|
|
8
|
+
export { createEmbeddingConfig, DEFAULT_EMBEDDING_CONFIG, createEmbedder, } from './embeddings/index.js';
|
|
9
|
+
export type { Embedder } from './embeddings/index.js';
|
|
10
|
+
export { createVectorStoreConfig, DEFAULT_VECTOR_STORE_CONFIG, createVectorStore, } from './vectors/index.js';
|
|
11
|
+
export type { VectorStore } from './vectors/index.js';
|
|
12
|
+
export { createRagConfig, DEFAULT_RAG_CONFIG, createRagPipeline, chunkText, } from './rag/index.js';
|
|
13
|
+
export type { RagPipeline } from './rag/index.js';
|
|
14
|
+
export { createAgentConfig, DEFAULT_AGENT_CONFIG, createAgent, defineTool, } from './agents/index.js';
|
|
15
|
+
export type { Agent } from './agents/index.js';
|
|
16
|
+
export { buildContextPrompt, tenantFilter, createContextAwareAgent, createContextAwareRag, } from './context/context.js';
|
|
17
|
+
export type { VentureKitAIContext, PermissionCheck, ContextAwareAgentOptions, ContextAwareRagOptions, } from './context/context.js';
|
|
18
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,cAAc,kBAAkB,CAAC;AAGjC,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,cAAc,GACf,MAAM,uBAAuB,CAAC;AAC/B,YAAY,EAAE,QAAQ,EAAE,MAAM,uBAAuB,CAAC;AAGtD,OAAO,EACL,uBAAuB,EACvB,2BAA2B,EAC3B,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAC5B,YAAY,EAAE,WAAW,EAAE,MAAM,oBAAoB,CAAC;AAGtD,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,SAAS,GACV,MAAM,gBAAgB,CAAC;AACxB,YAAY,EAAE,WAAW,EAAE,MAAM,gBAAgB,CAAC;AAGlD,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,WAAW,EACX,UAAU,GACX,MAAM,mBAAmB,CAAC;AAC3B,YAAY,EAAE,KAAK,EAAE,MAAM,mBAAmB,CAAC;AAG/C,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC;AAC9B,YAAY,EACV,mBAAmB,EACnB,eAAe,EACf,wBAAwB,EACxB,sBAAsB,GACvB,MAAM,sBAAsB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @venturekit-pro/ai
|
|
3
|
+
*
|
|
4
|
+
* AI utilities for VentureKit - embeddings, RAG, and agents.
|
|
5
|
+
* Documentation: https://venturekit.dev
|
|
6
|
+
*/
|
|
7
|
+
// Types
|
|
8
|
+
export * from './types/index.js';
|
|
9
|
+
// Embeddings
|
|
10
|
+
export { createEmbeddingConfig, DEFAULT_EMBEDDING_CONFIG, createEmbedder, } from './embeddings/index.js';
|
|
11
|
+
// Vector Store
|
|
12
|
+
export { createVectorStoreConfig, DEFAULT_VECTOR_STORE_CONFIG, createVectorStore, } from './vectors/index.js';
|
|
13
|
+
// RAG
|
|
14
|
+
export { createRagConfig, DEFAULT_RAG_CONFIG, createRagPipeline, chunkText, } from './rag/index.js';
|
|
15
|
+
// Agents
|
|
16
|
+
export { createAgentConfig, DEFAULT_AGENT_CONFIG, createAgent, defineTool, } from './agents/index.js';
|
|
17
|
+
// Context (tenant-aware, permission-aware AI)
|
|
18
|
+
export { buildContextPrompt, tenantFilter, createContextAwareAgent, createContextAwareRag, } from './context/context.js';
|
|
19
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,QAAQ;AACR,cAAc,kBAAkB,CAAC;AAEjC,aAAa;AACb,OAAO,EACL,qBAAqB,EACrB,wBAAwB,EACxB,cAAc,GACf,MAAM,uBAAuB,CAAC;AAG/B,eAAe;AACf,OAAO,EACL,uBAAuB,EACvB,2BAA2B,EAC3B,iBAAiB,GAClB,MAAM,oBAAoB,CAAC;AAG5B,MAAM;AACN,OAAO,EACL,eAAe,EACf,kBAAkB,EAClB,iBAAiB,EACjB,SAAS,GACV,MAAM,gBAAgB,CAAC;AAGxB,SAAS;AACT,OAAO,EACL,iBAAiB,EACjB,oBAAoB,EACpB,WAAW,EACX,UAAU,GACX,MAAM,mBAAmB,CAAC;AAG3B,8CAA8C;AAC9C,OAAO,EACL,kBAAkB,EAClB,YAAY,EACZ,uBAAuB,EACvB,qBAAqB,GACtB,MAAM,sBAAsB,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text Chunker
|
|
3
|
+
*
|
|
4
|
+
* Utilities for splitting text into chunks for embedding.
|
|
5
|
+
*/
|
|
6
|
+
import type { ChunkingConfig, DocumentChunk } from '../types/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* Chunk text into smaller pieces
|
|
9
|
+
*/
|
|
10
|
+
export declare function chunkText(text: string, sourceId: string, config: ChunkingConfig): DocumentChunk[];
|
|
11
|
+
//# sourceMappingURL=chunker.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.d.ts","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,mBAAmB,CAAC;AAEvE;;GAEG;AACH,wBAAgB,SAAS,CACvB,IAAI,EAAE,MAAM,EACZ,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,cAAc,GACrB,aAAa,EAAE,CAWjB"}
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Text Chunker
|
|
3
|
+
*
|
|
4
|
+
* Utilities for splitting text into chunks for embedding.
|
|
5
|
+
*/
|
|
6
|
+
/**
|
|
7
|
+
* Chunk text into smaller pieces
|
|
8
|
+
*/
|
|
9
|
+
export function chunkText(text, sourceId, config) {
|
|
10
|
+
switch (config.strategy) {
|
|
11
|
+
case 'fixed':
|
|
12
|
+
return fixedChunking(text, sourceId, config);
|
|
13
|
+
case 'sentence':
|
|
14
|
+
return sentenceChunking(text, sourceId, config);
|
|
15
|
+
case 'paragraph':
|
|
16
|
+
return paragraphChunking(text, sourceId, config);
|
|
17
|
+
default:
|
|
18
|
+
return fixedChunking(text, sourceId, config);
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
/**
|
|
22
|
+
* Fixed-size chunking with overlap
|
|
23
|
+
*/
|
|
24
|
+
function fixedChunking(text, sourceId, config) {
|
|
25
|
+
const chunks = [];
|
|
26
|
+
const { chunkSize, chunkOverlap } = config;
|
|
27
|
+
let start = 0;
|
|
28
|
+
let chunkIndex = 0;
|
|
29
|
+
while (start < text.length) {
|
|
30
|
+
const end = Math.min(start + chunkSize, text.length);
|
|
31
|
+
const content = text.slice(start, end);
|
|
32
|
+
chunks.push({
|
|
33
|
+
id: `${sourceId}-${chunkIndex}`,
|
|
34
|
+
content,
|
|
35
|
+
sourceId,
|
|
36
|
+
chunkIndex,
|
|
37
|
+
});
|
|
38
|
+
start = end - chunkOverlap;
|
|
39
|
+
if (start >= text.length - chunkOverlap)
|
|
40
|
+
break;
|
|
41
|
+
chunkIndex++;
|
|
42
|
+
}
|
|
43
|
+
return chunks;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Sentence-based chunking
|
|
47
|
+
*/
|
|
48
|
+
function sentenceChunking(text, sourceId, config) {
|
|
49
|
+
const sentences = text.match(/[^.!?]+[.!?]+/g) || [text];
|
|
50
|
+
const chunks = [];
|
|
51
|
+
const { chunkSize } = config;
|
|
52
|
+
let currentChunk = '';
|
|
53
|
+
let chunkIndex = 0;
|
|
54
|
+
for (const sentence of sentences) {
|
|
55
|
+
if (currentChunk.length + sentence.length > chunkSize && currentChunk) {
|
|
56
|
+
chunks.push({
|
|
57
|
+
id: `${sourceId}-${chunkIndex}`,
|
|
58
|
+
content: currentChunk.trim(),
|
|
59
|
+
sourceId,
|
|
60
|
+
chunkIndex,
|
|
61
|
+
});
|
|
62
|
+
currentChunk = '';
|
|
63
|
+
chunkIndex++;
|
|
64
|
+
}
|
|
65
|
+
currentChunk += sentence;
|
|
66
|
+
}
|
|
67
|
+
if (currentChunk.trim()) {
|
|
68
|
+
chunks.push({
|
|
69
|
+
id: `${sourceId}-${chunkIndex}`,
|
|
70
|
+
content: currentChunk.trim(),
|
|
71
|
+
sourceId,
|
|
72
|
+
chunkIndex,
|
|
73
|
+
});
|
|
74
|
+
}
|
|
75
|
+
return chunks;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Paragraph-based chunking
|
|
79
|
+
*/
|
|
80
|
+
function paragraphChunking(text, sourceId, config) {
|
|
81
|
+
const paragraphs = text.split(/\n\n+/);
|
|
82
|
+
const chunks = [];
|
|
83
|
+
const { chunkSize } = config;
|
|
84
|
+
let currentChunk = '';
|
|
85
|
+
let chunkIndex = 0;
|
|
86
|
+
for (const paragraph of paragraphs) {
|
|
87
|
+
if (currentChunk.length + paragraph.length > chunkSize && currentChunk) {
|
|
88
|
+
chunks.push({
|
|
89
|
+
id: `${sourceId}-${chunkIndex}`,
|
|
90
|
+
content: currentChunk.trim(),
|
|
91
|
+
sourceId,
|
|
92
|
+
chunkIndex,
|
|
93
|
+
});
|
|
94
|
+
currentChunk = '';
|
|
95
|
+
chunkIndex++;
|
|
96
|
+
}
|
|
97
|
+
currentChunk += paragraph + '\n\n';
|
|
98
|
+
}
|
|
99
|
+
if (currentChunk.trim()) {
|
|
100
|
+
chunks.push({
|
|
101
|
+
id: `${sourceId}-${chunkIndex}`,
|
|
102
|
+
content: currentChunk.trim(),
|
|
103
|
+
sourceId,
|
|
104
|
+
chunkIndex,
|
|
105
|
+
});
|
|
106
|
+
}
|
|
107
|
+
return chunks;
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/rag/chunker.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAIH;;GAEG;AACH,MAAM,UAAU,SAAS,CACvB,IAAY,EACZ,QAAgB,EAChB,MAAsB;IAEtB,QAAQ,MAAM,CAAC,QAAQ,EAAE,CAAC;QACxB,KAAK,OAAO;YACV,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAC/C,KAAK,UAAU;YACb,OAAO,gBAAgB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QAClD,KAAK,WAAW;YACd,OAAO,iBAAiB,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;QACnD;YACE,OAAO,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IACjD,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,aAAa,CACpB,IAAY,EACZ,QAAgB,EAChB,MAAsB;IAEtB,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,EAAE,SAAS,EAAE,YAAY,EAAE,GAAG,MAAM,CAAC;IAE3C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,OAAO,KAAK,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;QAC3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,GAAG,SAAS,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;QACrD,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QAEvC,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,QAAQ,IAAI,UAAU,EAAE;YAC/B,OAAO;YACP,QAAQ;YACR,UAAU;SACX,CAAC,CAAC;QAEH,KAAK,GAAG,GAAG,GAAG,YAAY,CAAC;QAC3B,IAAI,KAAK,IAAI,IAAI,CAAC,MAAM,GAAG,YAAY;YAAE,MAAM;QAC/C,UAAU,EAAE,CAAC;IACf,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,gBAAgB,CACvB,IAAY,EACZ,QAAgB,EAChB,MAAsB;IAEtB,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzD,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;IAE7B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,QAAQ,IAAI,SAAS,EAAE,CAAC;QACjC,IAAI,YAAY,CAAC,MAAM,GAAG,QAAQ,CAAC,MAAM,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;YACtE,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,QAAQ,IAAI,UAAU,EAAE;gBAC/B,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;gBAC5B,QAAQ;gBACR,UAAU;aACX,CAAC,CAAC;YACH,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,EAAE,CAAC;QACf,CAAC;QACD,YAAY,IAAI,QAAQ,CAAC;IAC3B,CAAC;IAED,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,QAAQ,IAAI,UAAU,EAAE;YAC/B,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;YAC5B,QAAQ;YACR,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CACxB,IAAY,EACZ,QAAgB,EAChB,MAAsB;IAEtB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IACvC,MAAM,MAAM,GAAoB,EAAE,CAAC;IACnC,MAAM,EAAE,SAAS,EAAE,GAAG,MAAM,CAAC;IAE7B,IAAI,YAAY,GAAG,EAAE,CAAC;IACtB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;QACnC,IAAI,YAAY,CAAC,MAAM,GAAG,SAAS,CAAC,MAAM,GAAG,SAAS,IAAI,YAAY,EAAE,CAAC;YACvE,MAAM,CAAC,IAAI,CAAC;gBACV,EAAE,EAAE,GAAG,QAAQ,IAAI,UAAU,EAAE;gBAC/B,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;gBAC5B,QAAQ;gBACR,UAAU;aACX,CAAC,CAAC;YACH,YAAY,GAAG,EAAE,CAAC;YAClB,UAAU,EAAE,CAAC;QACf,CAAC;QACD,YAAY,IAAI,SAAS,GAAG,MAAM,CAAC;IACrC,CAAC;IAED,IAAI,YAAY,CAAC,IAAI,EAAE,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,QAAQ,IAAI,UAAU,EAAE;YAC/B,OAAO,EAAE,YAAY,CAAC,IAAI,EAAE;YAC5B,QAAQ;YACR,UAAU;SACX,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Configuration
|
|
3
|
+
*/
|
|
4
|
+
import type { RagConfig, RagConfigInput } from '../types/index.js';
|
|
5
|
+
/**
|
|
6
|
+
* Default RAG configuration
|
|
7
|
+
*/
|
|
8
|
+
export declare const DEFAULT_RAG_CONFIG: RagConfig;
|
|
9
|
+
/**
|
|
10
|
+
* Create RAG configuration from input
|
|
11
|
+
*/
|
|
12
|
+
export declare function createRagConfig(input?: RagConfigInput): RagConfig;
|
|
13
|
+
//# sourceMappingURL=config.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/rag/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAInE;;GAEG;AACH,eAAO,MAAM,kBAAkB,EAAE,SAWhC,CAAC;AAEF;;GAEG;AACH,wBAAgB,eAAe,CAAC,KAAK,CAAC,EAAE,cAAc,GAAG,SAAS,CAUjE"}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Configuration
|
|
3
|
+
*/
|
|
4
|
+
import { DEFAULT_EMBEDDING_CONFIG } from '../embeddings/config.js';
|
|
5
|
+
import { DEFAULT_VECTOR_STORE_CONFIG } from '../vectors/config.js';
|
|
6
|
+
/**
|
|
7
|
+
* Default RAG configuration
|
|
8
|
+
*/
|
|
9
|
+
export const DEFAULT_RAG_CONFIG = {
|
|
10
|
+
embedding: DEFAULT_EMBEDDING_CONFIG,
|
|
11
|
+
vectorStore: DEFAULT_VECTOR_STORE_CONFIG,
|
|
12
|
+
chunking: {
|
|
13
|
+
strategy: 'fixed',
|
|
14
|
+
chunkSize: 1000,
|
|
15
|
+
chunkOverlap: 200,
|
|
16
|
+
},
|
|
17
|
+
topK: 5,
|
|
18
|
+
minScore: 0.7,
|
|
19
|
+
rerank: false,
|
|
20
|
+
};
|
|
21
|
+
/**
|
|
22
|
+
* Create RAG configuration from input
|
|
23
|
+
*/
|
|
24
|
+
export function createRagConfig(input) {
|
|
25
|
+
if (!input)
|
|
26
|
+
return DEFAULT_RAG_CONFIG;
|
|
27
|
+
return {
|
|
28
|
+
...DEFAULT_RAG_CONFIG,
|
|
29
|
+
...input,
|
|
30
|
+
embedding: { ...DEFAULT_RAG_CONFIG.embedding, ...input.embedding },
|
|
31
|
+
vectorStore: { ...DEFAULT_RAG_CONFIG.vectorStore, ...input.vectorStore },
|
|
32
|
+
chunking: { ...DEFAULT_RAG_CONFIG.chunking, ...input.chunking },
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/rag/config.ts"],"names":[],"mappings":"AAAA;;GAEG;AAGH,OAAO,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AACnE,OAAO,EAAE,2BAA2B,EAAE,MAAM,sBAAsB,CAAC;AAEnE;;GAEG;AACH,MAAM,CAAC,MAAM,kBAAkB,GAAc;IAC3C,SAAS,EAAE,wBAAwB;IACnC,WAAW,EAAE,2BAA2B;IACxC,QAAQ,EAAE;QACR,QAAQ,EAAE,OAAO;QACjB,SAAS,EAAE,IAAI;QACf,YAAY,EAAE,GAAG;KAClB;IACD,IAAI,EAAE,CAAC;IACP,QAAQ,EAAE,GAAG;IACb,MAAM,EAAE,KAAK;CACd,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,eAAe,CAAC,KAAsB;IACpD,IAAI,CAAC,KAAK;QAAE,OAAO,kBAAkB,CAAC;IAEtC,OAAO;QACL,GAAG,kBAAkB;QACrB,GAAG,KAAK;QACR,SAAS,EAAE,EAAE,GAAG,kBAAkB,CAAC,SAAS,EAAE,GAAG,KAAK,CAAC,SAAS,EAAE;QAClE,WAAW,EAAE,EAAE,GAAG,kBAAkB,CAAC,WAAW,EAAE,GAAG,KAAK,CAAC,WAAW,EAAE;QACxE,QAAQ,EAAE,EAAE,GAAG,kBAAkB,CAAC,QAAQ,EAAE,GAAG,KAAK,CAAC,QAAQ,EAAE;KAChE,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG (Retrieval-Augmented Generation)
|
|
3
|
+
*
|
|
4
|
+
* Utilities for building RAG pipelines.
|
|
5
|
+
*/
|
|
6
|
+
export { createRagConfig, DEFAULT_RAG_CONFIG } from './config.js';
|
|
7
|
+
export { createRagPipeline } from './pipeline.js';
|
|
8
|
+
export { chunkText } from './chunker.js';
|
|
9
|
+
export type { RagPipeline } from './pipeline.js';
|
|
10
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/rag/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC;AACzC,YAAY,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG (Retrieval-Augmented Generation)
|
|
3
|
+
*
|
|
4
|
+
* Utilities for building RAG pipelines.
|
|
5
|
+
*/
|
|
6
|
+
export { createRagConfig, DEFAULT_RAG_CONFIG } from './config.js';
|
|
7
|
+
export { createRagPipeline } from './pipeline.js';
|
|
8
|
+
export { chunkText } from './chunker.js';
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/rag/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,aAAa,CAAC;AAClE,OAAO,EAAE,iBAAiB,EAAE,MAAM,eAAe,CAAC;AAClD,OAAO,EAAE,SAAS,EAAE,MAAM,cAAc,CAAC"}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* RAG Pipeline
|
|
3
|
+
*
|
|
4
|
+
* End-to-end retrieval-augmented generation pipeline.
|
|
5
|
+
*/
|
|
6
|
+
import type { RagConfig, RetrievedContext, RagQueryOptions } from '../types/index.js';
|
|
7
|
+
/**
|
|
8
|
+
* RAG pipeline interface
|
|
9
|
+
*/
|
|
10
|
+
export interface RagPipeline {
|
|
11
|
+
/** Index documents */
|
|
12
|
+
index(documents: Array<{
|
|
13
|
+
id: string;
|
|
14
|
+
content: string;
|
|
15
|
+
metadata?: Record<string, unknown>;
|
|
16
|
+
}>): Promise<void>;
|
|
17
|
+
/** Retrieve relevant context for a query */
|
|
18
|
+
retrieve(query: string, options?: RagQueryOptions): Promise<RetrievedContext>;
|
|
19
|
+
/** Delete documents by ID */
|
|
20
|
+
delete(ids: string[]): Promise<void>;
|
|
21
|
+
}
|
|
22
|
+
/**
|
|
23
|
+
* Create a RAG pipeline
|
|
24
|
+
*/
|
|
25
|
+
export declare function createRagPipeline(config: RagConfig): RagPipeline;
|
|
26
|
+
//# sourceMappingURL=pipeline.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pipeline.d.ts","sourceRoot":"","sources":["../../src/rag/pipeline.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,SAAS,EAAiB,gBAAgB,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAKrG;;GAEG;AACH,MAAM,WAAW,WAAW;IAC1B,sBAAsB;IACtB,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAA;KAAE,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAE5G,4CAA4C;IAC5C,QAAQ,CAAC,KAAK,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,gBAAgB,CAAC,CAAC;IAE9E,6BAA6B;IAC7B,MAAM,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACtC;AAED;;GAEG;AACH,wBAAgB,iBAAiB,CAAC,MAAM,EAAE,SAAS,GAAG,WAAW,CA6EhE"}
|