@framers/agentos 0.1.22 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/core/workspace/AgentWorkspace.d.ts +35 -0
- package/dist/core/workspace/AgentWorkspace.d.ts.map +1 -0
- package/dist/core/workspace/AgentWorkspace.js +61 -0
- package/dist/core/workspace/AgentWorkspace.js.map +1 -0
- package/dist/core/workspace/index.d.ts +2 -0
- package/dist/core/workspace/index.d.ts.map +1 -0
- package/dist/core/workspace/index.js +2 -0
- package/dist/core/workspace/index.js.map +1 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/rag/IRetrievalAugmentor.d.ts +5 -0
- package/dist/rag/IRetrievalAugmentor.d.ts.map +1 -1
- package/dist/rag/IRetrievalAugmentor.js.map +1 -1
- package/dist/rag/RetrievalAugmentor.d.ts.map +1 -1
- package/dist/rag/RetrievalAugmentor.js +60 -2
- package/dist/rag/RetrievalAugmentor.js.map +1 -1
- package/dist/rag/audit/RAGAuditCollector.d.ts +97 -0
- package/dist/rag/audit/RAGAuditCollector.d.ts.map +1 -0
- package/dist/rag/audit/RAGAuditCollector.js +230 -0
- package/dist/rag/audit/RAGAuditCollector.js.map +1 -0
- package/dist/rag/audit/RAGAuditTypes.d.ts +108 -0
- package/dist/rag/audit/RAGAuditTypes.d.ts.map +1 -0
- package/dist/rag/audit/RAGAuditTypes.js +8 -0
- package/dist/rag/audit/RAGAuditTypes.js.map +1 -0
- package/dist/rag/audit/index.d.ts +8 -0
- package/dist/rag/audit/index.d.ts.map +1 -0
- package/dist/rag/audit/index.js +7 -0
- package/dist/rag/audit/index.js.map +1 -0
- package/dist/rag/graphrag/GraphRAGEngine.d.ts.map +1 -1
- package/dist/rag/graphrag/GraphRAGEngine.js +60 -0
- package/dist/rag/graphrag/GraphRAGEngine.js.map +1 -1
- package/dist/rag/graphrag/IGraphRAG.d.ts +3 -0
- package/dist/rag/graphrag/IGraphRAG.d.ts.map +1 -1
- package/dist/rag/index.d.ts +2 -0
- package/dist/rag/index.d.ts.map +1 -1
- package/dist/rag/index.js +1 -0
- package/dist/rag/index.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RAGAuditCollector.d.ts","sourceRoot":"","sources":["../../../src/rag/audit/RAGAuditCollector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,KAAK,EACV,aAAa,EACb,iBAAiB,EAElB,MAAM,oBAAoB,CAAC;AAO5B,4EAA4E;AAC5E,UAAU,eAAe;IACvB,WAAW,CACT,GAAG,EAAE;QAAE,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,UAAU,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAA;KAAE,EACrF,KAAK,EAAE;QAAE,YAAY,CAAC,EAAE,MAAM,CAAC;QAAC,gBAAgB,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,MAAM,CAAC;QAAC,OAAO,CAAC,EAAE,OAAO,CAAA;KAAE,GACrH,IAAI,CAAC;CACT;AAED,MAAM,WAAW,wBAAwB;IACvC,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,qGAAqG;IACrG,WAAW,CAAC,EAAE,eAAe,CAAC;CAC/B;AAED;;;GAGG;AACH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAoB;IAC1C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,SAAS,CAAS;IAC1B,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAqC;gBAG9D,IAAI,EAAE,iBAAiB,CAAC,eAAe,CAAC,EACxC,UAAU,EAAE,CAAC,KAAK,EAAE,iBAAiB,KAAK,IAAI;IAqBhD,kBAAkB,CAAC,MAAM,EAAE,iBAAiB,CAAC,iBAAiB,CAAC,GAAG,IAAI;IAKtE,UAAU,CACR,MAAM,EAAE,KAAK,CAAC;QACZ,EAAE,CAAC,EAAE,MAAM,CAAC;QACZ,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,kBAAkB,CAAC,EAAE,MAAM,CAAC;QAC5B,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,OAAO,CAAC,EAAE,MAAM,CAAC;QACjB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,cAAc,CAAC,EAAE,MAAM,CAAC;QACxB,YAAY,CAAC,EAAE,MAAM,CAAC;QACtB,MAAM,CAAC,EAAE,MAAM,CAAC;QAChB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;KACpC,CAAC,GACD,IAAI;IAiBP,aAAa,CAAC,KAAK,EAAE,iBAAiB,CAAC,YAAY,CAAC,GAAG,IAAI;IAK3D,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI;IAK9B,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI;IAKrC,gBAAgB,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,IAAI;IAKrC,eAAe,CAAC,OAAO,EAAE,WAAW,CAAC,iBAAiB,CAAC,cAAc,CAAC,CAAC,GAAG,IAAI;IAK9E,gBAAgB,CAAC,OAAO,EAAE,WAAW,CAAC,iBAAiB,CAAC,eAAe,CAAC,CAAC,GAAG,IAAI;IAKhF;;;;;;OAMG;IACH,QAAQ,CAAC,YAAY,EAAE,MAAM,EAAE,kBAAkB,CAAC,EAAE,MAAM,GAAG,iBAAiB;CAoB/E;AAED;;;GAGG;AACH,qBAAa,iBAAiB;IAC5B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA2B;gBAE1C,OAAO,EAAE,wBAAwB;IAM7C,+DAA+D;IAC/D,cAAc,CAAC,IAAI,EAAE,iBAAiB,CAAC,eAAe,CAAC,GAAG,kBAAkB;IAM5E,0EAA0E;IAC1E,QAAQ,IAAI,aAAa;CAyG1B"}
|
|
@@ -0,0 +1,230 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Request-scoped collector that accumulates RAG operations during
|
|
3
|
+
* a single retrieval pipeline execution and produces a finalized RAGAuditTrail.
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* ```typescript
|
|
7
|
+
* const collector = new RAGAuditCollector({ requestId: 'req-1', query: 'What is ML?' });
|
|
8
|
+
*
|
|
9
|
+
* const embedOp = collector.startOperation('embedding');
|
|
10
|
+
* // ... do embedding work ...
|
|
11
|
+
* embedOp.setTokenUsage({ embeddingTokens: 512, llmPromptTokens: 0, llmCompletionTokens: 0, totalTokens: 512 });
|
|
12
|
+
* embedOp.complete(1);
|
|
13
|
+
*
|
|
14
|
+
* const trail = collector.finalize();
|
|
15
|
+
* ```
|
|
16
|
+
*
|
|
17
|
+
* @module @framers/agentos/rag/audit
|
|
18
|
+
*/
|
|
19
|
+
let _idCounter = 0;
|
|
20
|
+
function nextId(prefix) {
|
|
21
|
+
return `${prefix}-${Date.now().toString(36)}-${(++_idCounter).toString(36)}`;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Fluent handle for a single in-flight RAG operation.
|
|
25
|
+
* Call `.complete(resultsCount)` to finalize timing and add it to the collector.
|
|
26
|
+
*/
|
|
27
|
+
export class RAGOperationHandle {
|
|
28
|
+
constructor(type, onComplete) {
|
|
29
|
+
this.completed = false;
|
|
30
|
+
this.startTime = Date.now();
|
|
31
|
+
this.onComplete = onComplete;
|
|
32
|
+
this.entry = {
|
|
33
|
+
operationId: nextId(`op-${type}`),
|
|
34
|
+
operationType: type,
|
|
35
|
+
startedAt: new Date().toISOString(),
|
|
36
|
+
durationMs: 0,
|
|
37
|
+
sources: [],
|
|
38
|
+
tokenUsage: {
|
|
39
|
+
embeddingTokens: 0,
|
|
40
|
+
llmPromptTokens: 0,
|
|
41
|
+
llmCompletionTokens: 0,
|
|
42
|
+
totalTokens: 0,
|
|
43
|
+
},
|
|
44
|
+
costUSD: 0,
|
|
45
|
+
resultsCount: 0,
|
|
46
|
+
};
|
|
47
|
+
}
|
|
48
|
+
setRetrievalMethod(method) {
|
|
49
|
+
this.entry.retrievalMethod = method;
|
|
50
|
+
return this;
|
|
51
|
+
}
|
|
52
|
+
addSources(chunks) {
|
|
53
|
+
for (const c of chunks) {
|
|
54
|
+
const chunkId = c.chunkId ?? c.id ?? 'unknown';
|
|
55
|
+
const documentId = c.documentId ?? c.originalDocumentId ?? chunkId;
|
|
56
|
+
this.entry.sources.push({
|
|
57
|
+
chunkId,
|
|
58
|
+
documentId,
|
|
59
|
+
source: c.source,
|
|
60
|
+
contentSnippet: c.contentSnippet ?? (c.content ?? '').slice(0, 200),
|
|
61
|
+
relevanceScore: c.relevanceScore ?? 0,
|
|
62
|
+
dataSourceId: c.dataSourceId,
|
|
63
|
+
metadata: c.metadata,
|
|
64
|
+
});
|
|
65
|
+
}
|
|
66
|
+
return this;
|
|
67
|
+
}
|
|
68
|
+
setTokenUsage(usage) {
|
|
69
|
+
this.entry.tokenUsage = usage;
|
|
70
|
+
return this;
|
|
71
|
+
}
|
|
72
|
+
setCost(costUSD) {
|
|
73
|
+
this.entry.costUSD = costUSD;
|
|
74
|
+
return this;
|
|
75
|
+
}
|
|
76
|
+
setDataSourceIds(ids) {
|
|
77
|
+
this.entry.dataSourceIds = ids;
|
|
78
|
+
return this;
|
|
79
|
+
}
|
|
80
|
+
setCollectionIds(ids) {
|
|
81
|
+
this.entry.collectionIds = ids;
|
|
82
|
+
return this;
|
|
83
|
+
}
|
|
84
|
+
setGraphDetails(details) {
|
|
85
|
+
this.entry.graphDetails = details;
|
|
86
|
+
return this;
|
|
87
|
+
}
|
|
88
|
+
setRerankDetails(details) {
|
|
89
|
+
this.entry.rerankDetails = details;
|
|
90
|
+
return this;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* Finalizes the operation, records duration, computes relevance score stats,
|
|
94
|
+
* and adds the entry to the parent collector.
|
|
95
|
+
*
|
|
96
|
+
* @param resultsCount Number of results this operation produced.
|
|
97
|
+
* @param overrideDurationMs Optional override for duration (when timing is measured externally).
|
|
98
|
+
*/
|
|
99
|
+
complete(resultsCount, overrideDurationMs) {
|
|
100
|
+
if (this.completed)
|
|
101
|
+
return this.entry;
|
|
102
|
+
this.completed = true;
|
|
103
|
+
this.entry.durationMs = overrideDurationMs ?? (Date.now() - this.startTime);
|
|
104
|
+
this.entry.resultsCount = resultsCount;
|
|
105
|
+
// Compute relevance score stats from sources
|
|
106
|
+
if (this.entry.sources.length > 0) {
|
|
107
|
+
const scores = this.entry.sources.map((s) => s.relevanceScore);
|
|
108
|
+
this.entry.relevanceScores = {
|
|
109
|
+
min: Math.min(...scores),
|
|
110
|
+
max: Math.max(...scores),
|
|
111
|
+
avg: scores.reduce((a, b) => a + b, 0) / scores.length,
|
|
112
|
+
};
|
|
113
|
+
}
|
|
114
|
+
this.onComplete(this.entry);
|
|
115
|
+
return this.entry;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
/**
|
|
119
|
+
* Request-scoped audit collector. Create one per `retrieveContext()` call.
|
|
120
|
+
* NOT a singleton — scoped to a single pipeline execution.
|
|
121
|
+
*/
|
|
122
|
+
export class RAGAuditCollector {
|
|
123
|
+
constructor(options) {
|
|
124
|
+
this.operations = [];
|
|
125
|
+
this.options = options;
|
|
126
|
+
this.trailId = nextId('trail');
|
|
127
|
+
this.startTime = Date.now();
|
|
128
|
+
}
|
|
129
|
+
/** Start tracking a new operation. Returns a fluent handle. */
|
|
130
|
+
startOperation(type) {
|
|
131
|
+
return new RAGOperationHandle(type, (entry) => {
|
|
132
|
+
this.operations.push(entry);
|
|
133
|
+
});
|
|
134
|
+
}
|
|
135
|
+
/** Finalize and return the complete audit trail with computed summary. */
|
|
136
|
+
finalize() {
|
|
137
|
+
const totalDurationMs = Date.now() - this.startTime;
|
|
138
|
+
// Aggregate totals
|
|
139
|
+
let totalTokens = 0;
|
|
140
|
+
let totalPromptTokens = 0;
|
|
141
|
+
let totalCompletionTokens = 0;
|
|
142
|
+
let totalEmbeddingTokens = 0;
|
|
143
|
+
let totalCostUSD = 0;
|
|
144
|
+
let totalLLMCalls = 0;
|
|
145
|
+
let totalEmbeddingCalls = 0;
|
|
146
|
+
const operationTypes = new Set();
|
|
147
|
+
const uniqueDocuments = new Set();
|
|
148
|
+
const uniqueCollections = new Set();
|
|
149
|
+
const uniqueDataSources = new Set();
|
|
150
|
+
for (const op of this.operations) {
|
|
151
|
+
operationTypes.add(op.operationType);
|
|
152
|
+
totalTokens += op.tokenUsage.totalTokens;
|
|
153
|
+
totalPromptTokens += op.tokenUsage.llmPromptTokens;
|
|
154
|
+
totalCompletionTokens += op.tokenUsage.llmCompletionTokens;
|
|
155
|
+
totalEmbeddingTokens += op.tokenUsage.embeddingTokens;
|
|
156
|
+
totalCostUSD += op.costUSD;
|
|
157
|
+
if (op.operationType === 'embedding') {
|
|
158
|
+
totalEmbeddingCalls++;
|
|
159
|
+
}
|
|
160
|
+
else if (op.operationType === 'rerank' ||
|
|
161
|
+
op.operationType === 'graph_local' ||
|
|
162
|
+
op.operationType === 'graph_global') {
|
|
163
|
+
// Graph search and reranking use LLM calls
|
|
164
|
+
if (op.tokenUsage.llmPromptTokens > 0 || op.tokenUsage.llmCompletionTokens > 0) {
|
|
165
|
+
totalLLMCalls++;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
for (const src of op.sources) {
|
|
169
|
+
uniqueDocuments.add(src.documentId);
|
|
170
|
+
if (src.dataSourceId)
|
|
171
|
+
uniqueDataSources.add(src.dataSourceId);
|
|
172
|
+
}
|
|
173
|
+
if (op.collectionIds) {
|
|
174
|
+
for (const cId of op.collectionIds)
|
|
175
|
+
uniqueCollections.add(cId);
|
|
176
|
+
}
|
|
177
|
+
if (op.dataSourceIds) {
|
|
178
|
+
for (const dsId of op.dataSourceIds)
|
|
179
|
+
uniqueDataSources.add(dsId);
|
|
180
|
+
}
|
|
181
|
+
}
|
|
182
|
+
const trail = {
|
|
183
|
+
trailId: this.trailId,
|
|
184
|
+
requestId: this.options.requestId,
|
|
185
|
+
seedId: this.options.seedId,
|
|
186
|
+
sessionId: this.options.sessionId,
|
|
187
|
+
query: this.options.query,
|
|
188
|
+
timestamp: new Date(this.startTime).toISOString(),
|
|
189
|
+
operations: this.operations,
|
|
190
|
+
summary: {
|
|
191
|
+
totalOperations: this.operations.length,
|
|
192
|
+
totalLLMCalls,
|
|
193
|
+
totalEmbeddingCalls,
|
|
194
|
+
totalTokens,
|
|
195
|
+
totalPromptTokens,
|
|
196
|
+
totalCompletionTokens,
|
|
197
|
+
totalEmbeddingTokens,
|
|
198
|
+
totalCostUSD,
|
|
199
|
+
totalDurationMs,
|
|
200
|
+
operationTypes: Array.from(operationTypes),
|
|
201
|
+
sourceSummary: {
|
|
202
|
+
uniqueDocuments: uniqueDocuments.size,
|
|
203
|
+
uniqueCollections: uniqueCollections.size,
|
|
204
|
+
uniqueDataSources: uniqueDataSources.size,
|
|
205
|
+
},
|
|
206
|
+
},
|
|
207
|
+
};
|
|
208
|
+
// Push to UsageLedger when one was provided.
|
|
209
|
+
if (this.options.usageLedger && this.options.sessionId) {
|
|
210
|
+
const ledger = this.options.usageLedger;
|
|
211
|
+
const sessionId = this.options.sessionId;
|
|
212
|
+
const personaId = this.options.seedId;
|
|
213
|
+
for (const op of this.operations) {
|
|
214
|
+
const providerId = op.operationType === 'embedding' ? 'rag-embedding'
|
|
215
|
+
: op.operationType === 'rerank' ? 'rag-rerank'
|
|
216
|
+
: op.operationType.startsWith('graph_') ? 'rag-graphrag'
|
|
217
|
+
: 'rag-vector';
|
|
218
|
+
ledger.ingestUsage({ sessionId, personaId, providerId, modelId: op.rerankDetails?.modelId }, {
|
|
219
|
+
promptTokens: op.tokenUsage.llmPromptTokens,
|
|
220
|
+
completionTokens: op.tokenUsage.llmCompletionTokens,
|
|
221
|
+
totalTokens: op.tokenUsage.totalTokens,
|
|
222
|
+
costUSD: op.costUSD || undefined,
|
|
223
|
+
isFinal: true,
|
|
224
|
+
});
|
|
225
|
+
}
|
|
226
|
+
}
|
|
227
|
+
return trail;
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
//# sourceMappingURL=RAGAuditCollector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RAGAuditCollector.js","sourceRoot":"","sources":["../../../src/rag/audit/RAGAuditCollector.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAQH,IAAI,UAAU,GAAG,CAAC,CAAC;AACnB,SAAS,MAAM,CAAC,MAAc;IAC5B,OAAO,GAAG,MAAM,IAAI,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC;AAC/E,CAAC;AAmBD;;;GAGG;AACH,MAAM,OAAO,kBAAkB;IAM7B,YACE,IAAwC,EACxC,UAA8C;QALxC,cAAS,GAAG,KAAK,CAAC;QAOxB,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC5B,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;QAC7B,IAAI,CAAC,KAAK,GAAG;YACX,WAAW,EAAE,MAAM,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,aAAa,EAAE,IAAI;YACnB,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YACnC,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,EAAE;YACX,UAAU,EAAE;gBACV,eAAe,EAAE,CAAC;gBAClB,eAAe,EAAE,CAAC;gBAClB,mBAAmB,EAAE,CAAC;gBACtB,WAAW,EAAE,CAAC;aACf;YACD,OAAO,EAAE,CAAC;YACV,YAAY,EAAE,CAAC;SAChB,CAAC;IACJ,CAAC;IAED,kBAAkB,CAAC,MAA4C;QAC7D,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG,MAAM,CAAC;QACpC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,UAAU,CACR,MAWE;QAEF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;YACvB,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,EAAE,IAAI,SAAS,CAAC;YAC/C,MAAM,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC,kBAAkB,IAAI,OAAO,CAAC;YACnE,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;gBACtB,OAAO;gBACP,UAAU;gBACV,MAAM,EAAE,CAAC,CAAC,MAAM;gBAChB,cAAc,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;gBACnE,cAAc,EAAE,CAAC,CAAC,cAAc,IAAI,CAAC;gBACrC,YAAY,EAAE,CAAC,CAAC,YAAY;gBAC5B,QAAQ,EAAE,CAAC,CAAC,QAAQ;aACrB,CAAC,CAAC;QACL,CAAC;QACD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,aAAa,CAAC,KAAsC;QAClD,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,KAAK,CAAC;QAC9B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO,CAAC,OAAe;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,GAAG,OAAO,CAAC;QAC7B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gBAAgB,CAAC,GAAa;QAC5B,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gBAAgB,CAAC,GAAa;QAC5B,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;QAC/B,OAAO,IAAI,CAAC;IACd,CAAC;IAED,eAAe,CAAC,OAAuD;QACrE,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,OAAO,CAAC;QAClC,OAAO,IAAI,CAAC;IACd,CAAC;IAED,gBAAgB,CAAC,OAAwD;QACvE,IAAI,CAAC,KAAK,CAAC,aAAa,GAAG,OAAO,CAAC;QACnC,OAAO,IAAI,CAAC;IACd,CAAC;IAED;;;;;;OAMG;IACH,QAAQ,CAAC,YAAoB,EAAE,kBAA2B;QACxD,IAAI,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC,KAAK,CAAC;QACtC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC;QAEtB,IAAI,CAAC,KAAK,CAAC,UAAU,GAAG,kBAAkB,IAAI,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC;QAC5E,IAAI,CAAC,KAAK,CAAC,YAAY,GAAG,YAAY,CAAC;QAEvC,6CAA6C;QAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,CAAC,CAAC;YAC/D,IAAI,CAAC,KAAK,CAAC,eAAe,GAAG;gBAC3B,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;gBACxB,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,GAAG,MAAM,CAAC;gBACxB,GAAG,EAAE,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC,MAAM;aACvD,CAAC;QACJ,CAAC;QAED,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC;IACpB,CAAC;CACF;AAED;;;GAGG;AACH,MAAM,OAAO,iBAAiB;IAM5B,YAAY,OAAiC;QAF5B,eAAU,GAAwB,EAAE,CAAC;QAGpD,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAED,+DAA+D;IAC/D,cAAc,CAAC,IAAwC;QACrD,OAAO,IAAI,kBAAkB,CAAC,IAAI,EAAE,CAAC,KAAK,EAAE,EAAE;YAC5C,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;IACL,CAAC;IAED,0EAA0E;IAC1E,QAAQ;QACN,MAAM,eAAe,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAEpD,mBAAmB;QACnB,IAAI,WAAW,GAAG,CAAC,CAAC;QACpB,IAAI,iBAAiB,GAAG,CAAC,CAAC;QAC1B,IAAI,qBAAqB,GAAG,CAAC,CAAC;QAC9B,IAAI,oBAAoB,GAAG,CAAC,CAAC;QAC7B,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,IAAI,aAAa,GAAG,CAAC,CAAC;QACtB,IAAI,mBAAmB,GAAG,CAAC,CAAC;QAC5B,MAAM,cAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QACzC,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;QAC1C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAC5C,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAU,CAAC;QAE5C,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;YACjC,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,aAAa,CAAC,CAAC;YAErC,WAAW,IAAI,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC;YACzC,iBAAiB,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC;YACnD,qBAAqB,IAAI,EAAE,CAAC,UAAU,CAAC,mBAAmB,CAAC;YAC3D,oBAAoB,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,CAAC;YACtD,YAAY,IAAI,EAAE,CAAC,OAAO,CAAC;YAE3B,IAAI,EAAE,CAAC,aAAa,KAAK,WAAW,EAAE,CAAC;gBACrC,mBAAmB,EAAE,CAAC;YACxB,CAAC;iBAAM,IACL,EAAE,CAAC,aAAa,KAAK,QAAQ;gBAC7B,EAAE,CAAC,aAAa,KAAK,aAAa;gBAClC,EAAE,CAAC,aAAa,KAAK,cAAc,EACnC,CAAC;gBACD,2CAA2C;gBAC3C,IAAI,EAAE,CAAC,UAAU,CAAC,eAAe,GAAG,CAAC,IAAI,EAAE,CAAC,UAAU,CAAC,mBAAmB,GAAG,CAAC,EAAE,CAAC;oBAC/E,aAAa,EAAE,CAAC;gBAClB,CAAC;YACH,CAAC;YAED,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,OAAO,EAAE,CAAC;gBAC7B,eAAe,CAAC,GAAG,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBACpC,IAAI,GAAG,CAAC,YAAY;oBAAE,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YAChE,CAAC;YACD,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC;gBACrB,KAAK,MAAM,GAAG,IAAI,EAAE,CAAC,aAAa;oBAAE,iBAAiB,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACjE,CAAC;YACD,IAAI,EAAE,CAAC,aAAa,EAAE,CAAC;gBACrB,KAAK,MAAM,IAAI,IAAI,EAAE,CAAC,aAAa;oBAAE,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;YACnE,CAAC;QACH,CAAC;QAED,MAAM,KAAK,GAAkB;YAC3B,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YACjC,MAAM,EAAE,IAAI,CAAC,OAAO,CAAC,MAAM;YAC3B,SAAS,EAAE,IAAI,CAAC,OAAO,CAAC,SAAS;YACjC,KAAK,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK;YACzB,SAAS,EAAE,IAAI,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,WAAW,EAAE;YACjD,UAAU,EAAE,IAAI,CAAC,UAAU;YAC3B,OAAO,EAAE;gBACP,eAAe,EAAE,IAAI,CAAC,UAAU,CAAC,MAAM;gBACvC,aAAa;gBACb,mBAAmB;gBACnB,WAAW;gBACX,iBAAiB;gBACjB,qBAAqB;gBACrB,oBAAoB;gBACpB,YAAY;gBACZ,eAAe;gBACf,cAAc,EAAE,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC;gBAC1C,aAAa,EAAE;oBACb,eAAe,EAAE,eAAe,CAAC,IAAI;oBACrC,iBAAiB,EAAE,iBAAiB,CAAC,IAAI;oBACzC,iBAAiB,EAAE,iBAAiB,CAAC,IAAI;iBAC1C;aACF;SACF,CAAC;QAEF,6CAA6C;QAC7C,IAAI,IAAI,CAAC,OAAO,CAAC,WAAW,IAAI,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;YACvD,MAAM,MAAM,GAAG,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;YACxC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC;YACzC,MAAM,SAAS,GAAG,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC;YAEtC,KAAK,MAAM,EAAE,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBACjC,MAAM,UAAU,GACd,EAAE,CAAC,aAAa,KAAK,WAAW,CAAC,CAAC,CAAC,eAAe;oBAClD,CAAC,CAAC,EAAE,CAAC,aAAa,KAAK,QAAQ,CAAC,CAAC,CAAC,YAAY;wBAC9C,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc;4BACxD,CAAC,CAAC,YAAY,CAAC;gBAEjB,MAAM,CAAC,WAAW,CAChB,EAAE,SAAS,EAAE,SAAS,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC,aAAa,EAAE,OAAO,EAAE,EACxE;oBACE,YAAY,EAAE,EAAE,CAAC,UAAU,CAAC,eAAe;oBAC3C,gBAAgB,EAAE,EAAE,CAAC,UAAU,CAAC,mBAAmB;oBACnD,WAAW,EAAE,EAAE,CAAC,UAAU,CAAC,WAAW;oBACtC,OAAO,EAAE,EAAE,CAAC,OAAO,IAAI,SAAS;oBAChC,OAAO,EAAE,IAAI;iBACd,CACF,CAAC;YACJ,CAAC;QACH,CAAC;QAED,OAAO,KAAK,CAAC;IACf,CAAC;CACF"}
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Types for RAG audit trails — full transparency on memory/RAG operations.
|
|
3
|
+
* Captures per-operation details (vector search, graph search, reranking, embeddings)
|
|
4
|
+
* with token usage, cost tracking, and source attribution.
|
|
5
|
+
* @module @framers/agentos/rag/audit
|
|
6
|
+
*/
|
|
7
|
+
/** A single RAG operation within a request pipeline. */
|
|
8
|
+
export interface RAGOperationEntry {
|
|
9
|
+
operationId: string;
|
|
10
|
+
operationType: 'vector_query' | 'graph_local' | 'graph_global' | 'ingest' | 'rerank' | 'embedding';
|
|
11
|
+
/** ISO 8601 timestamp when the operation started. */
|
|
12
|
+
startedAt: string;
|
|
13
|
+
/** Duration in milliseconds. */
|
|
14
|
+
durationMs: number;
|
|
15
|
+
/** Retrieval strategy details (for vector_query operations). */
|
|
16
|
+
retrievalMethod?: {
|
|
17
|
+
strategy: 'similarity' | 'mmr' | 'hybrid';
|
|
18
|
+
hybridAlpha?: number;
|
|
19
|
+
topK?: number;
|
|
20
|
+
mmrLambda?: number;
|
|
21
|
+
};
|
|
22
|
+
/** Source documents/chunks that contributed to this operation's results. */
|
|
23
|
+
sources: RAGSourceAttribution[];
|
|
24
|
+
/** Token usage breakdown for this operation. */
|
|
25
|
+
tokenUsage: {
|
|
26
|
+
embeddingTokens: number;
|
|
27
|
+
llmPromptTokens: number;
|
|
28
|
+
llmCompletionTokens: number;
|
|
29
|
+
totalTokens: number;
|
|
30
|
+
};
|
|
31
|
+
/** Estimated cost in USD for this operation. */
|
|
32
|
+
costUSD: number;
|
|
33
|
+
/** Number of results returned by this operation. */
|
|
34
|
+
resultsCount: number;
|
|
35
|
+
/** Relevance score statistics across results. */
|
|
36
|
+
relevanceScores?: {
|
|
37
|
+
min: number;
|
|
38
|
+
max: number;
|
|
39
|
+
avg: number;
|
|
40
|
+
};
|
|
41
|
+
/** Data source IDs queried by this operation. */
|
|
42
|
+
dataSourceIds?: string[];
|
|
43
|
+
/** Collection IDs involved in this operation. */
|
|
44
|
+
collectionIds?: string[];
|
|
45
|
+
/** Graph-specific details (for graph_local / graph_global operations). */
|
|
46
|
+
graphDetails?: {
|
|
47
|
+
entitiesMatched: number;
|
|
48
|
+
communitiesSearched: number;
|
|
49
|
+
traversalTimeMs: number;
|
|
50
|
+
};
|
|
51
|
+
/** Reranking-specific details (for rerank operations). */
|
|
52
|
+
rerankDetails?: {
|
|
53
|
+
providerId: string;
|
|
54
|
+
modelId: string;
|
|
55
|
+
documentsReranked: number;
|
|
56
|
+
};
|
|
57
|
+
}
|
|
58
|
+
/** Attribution to a specific source document/chunk. */
|
|
59
|
+
export interface RAGSourceAttribution {
|
|
60
|
+
chunkId: string;
|
|
61
|
+
documentId: string;
|
|
62
|
+
/** Original source pointer (URL, file path, etc.). */
|
|
63
|
+
source?: string;
|
|
64
|
+
/** First 200 characters of chunk content. */
|
|
65
|
+
contentSnippet: string;
|
|
66
|
+
/** Similarity/relevance score (0–1). */
|
|
67
|
+
relevanceScore: number;
|
|
68
|
+
/** Data source / collection this chunk came from. */
|
|
69
|
+
dataSourceId?: string;
|
|
70
|
+
/** Chunk metadata. */
|
|
71
|
+
metadata?: Record<string, unknown>;
|
|
72
|
+
}
|
|
73
|
+
/** Aggregated audit trail for a complete RAG request. */
|
|
74
|
+
export interface RAGAuditTrail {
|
|
75
|
+
trailId: string;
|
|
76
|
+
/** Correlates with the conversation turn or API request. */
|
|
77
|
+
requestId: string;
|
|
78
|
+
/** Wunderland agent seed ID. */
|
|
79
|
+
seedId?: string;
|
|
80
|
+
/** Conversation session ID. */
|
|
81
|
+
sessionId?: string;
|
|
82
|
+
/** The user query that triggered RAG. */
|
|
83
|
+
query: string;
|
|
84
|
+
/** ISO 8601 timestamp. */
|
|
85
|
+
timestamp: string;
|
|
86
|
+
/** Per-operation breakdown. */
|
|
87
|
+
operations: RAGOperationEntry[];
|
|
88
|
+
/** Aggregated summary across all operations. */
|
|
89
|
+
summary: {
|
|
90
|
+
totalOperations: number;
|
|
91
|
+
totalLLMCalls: number;
|
|
92
|
+
totalEmbeddingCalls: number;
|
|
93
|
+
totalTokens: number;
|
|
94
|
+
totalPromptTokens: number;
|
|
95
|
+
totalCompletionTokens: number;
|
|
96
|
+
totalEmbeddingTokens: number;
|
|
97
|
+
totalCostUSD: number;
|
|
98
|
+
totalDurationMs: number;
|
|
99
|
+
/** Unique operation types used (e.g. ['embedding', 'vector_query', 'rerank']). */
|
|
100
|
+
operationTypes: string[];
|
|
101
|
+
sourceSummary: {
|
|
102
|
+
uniqueDocuments: number;
|
|
103
|
+
uniqueCollections: number;
|
|
104
|
+
uniqueDataSources: number;
|
|
105
|
+
};
|
|
106
|
+
};
|
|
107
|
+
}
|
|
108
|
+
//# sourceMappingURL=RAGAuditTypes.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RAGAuditTypes.d.ts","sourceRoot":"","sources":["../../../src/rag/audit/RAGAuditTypes.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,wDAAwD;AACxD,MAAM,WAAW,iBAAiB;IAChC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EACT,cAAc,GACd,aAAa,GACb,cAAc,GACd,QAAQ,GACR,QAAQ,GACR,WAAW,CAAC;IAEhB,qDAAqD;IACrD,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,UAAU,EAAE,MAAM,CAAC;IAEnB,gEAAgE;IAChE,eAAe,CAAC,EAAE;QAChB,QAAQ,EAAE,YAAY,GAAG,KAAK,GAAG,QAAQ,CAAC;QAC1C,WAAW,CAAC,EAAE,MAAM,CAAC;QACrB,IAAI,CAAC,EAAE,MAAM,CAAC;QACd,SAAS,CAAC,EAAE,MAAM,CAAC;KACpB,CAAC;IAEF,4EAA4E;IAC5E,OAAO,EAAE,oBAAoB,EAAE,CAAC;IAEhC,gDAAgD;IAChD,UAAU,EAAE;QACV,eAAe,EAAE,MAAM,CAAC;QACxB,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC;KACrB,CAAC;IAEF,gDAAgD;IAChD,OAAO,EAAE,MAAM,CAAC;IAEhB,oDAAoD;IACpD,YAAY,EAAE,MAAM,CAAC;IAErB,iDAAiD;IACjD,eAAe,CAAC,EAAE;QAAE,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,MAAM,CAAA;KAAE,CAAC;IAE5D,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB,iDAAiD;IACjD,aAAa,CAAC,EAAE,MAAM,EAAE,CAAC;IAEzB,0EAA0E;IAC1E,YAAY,CAAC,EAAE;QACb,eAAe,EAAE,MAAM,CAAC;QACxB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,eAAe,EAAE,MAAM,CAAC;KACzB,CAAC;IAEF,0DAA0D;IAC1D,aAAa,CAAC,EAAE;QACd,UAAU,EAAE,MAAM,CAAC;QACnB,OAAO,EAAE,MAAM,CAAC;QAChB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;CACH;AAED,uDAAuD;AACvD,MAAM,WAAW,oBAAoB;IACnC,OAAO,EAAE,MAAM,CAAC;IAChB,UAAU,EAAE,MAAM,CAAC;IACnB,sDAAsD;IACtD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,6CAA6C;IAC7C,cAAc,EAAE,MAAM,CAAC;IACvB,wCAAwC;IACxC,cAAc,EAAE,MAAM,CAAC;IACvB,qDAAqD;IACrD,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sBAAsB;IACtB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC;AAED,yDAAyD;AACzD,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,4DAA4D;IAC5D,SAAS,EAAE,MAAM,CAAC;IAClB,gCAAgC;IAChC,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,+BAA+B;IAC/B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,yCAAyC;IACzC,KAAK,EAAE,MAAM,CAAC;IACd,0BAA0B;IAC1B,SAAS,EAAE,MAAM,CAAC;IAElB,+BAA+B;IAC/B,UAAU,EAAE,iBAAiB,EAAE,CAAC;IAEhC,gDAAgD;IAChD,OAAO,EAAE;QACP,eAAe,EAAE,MAAM,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,mBAAmB,EAAE,MAAM,CAAC;QAC5B,WAAW,EAAE,MAAM,CAAC;QACpB,iBAAiB,EAAE,MAAM,CAAC;QAC1B,qBAAqB,EAAE,MAAM,CAAC;QAC9B,oBAAoB,EAAE,MAAM,CAAC;QAC7B,YAAY,EAAE,MAAM,CAAC;QACrB,eAAe,EAAE,MAAM,CAAC;QACxB,kFAAkF;QAClF,cAAc,EAAE,MAAM,EAAE,CAAC;QACzB,aAAa,EAAE;YACb,eAAe,EAAE,MAAM,CAAC;YACxB,iBAAiB,EAAE,MAAM,CAAC;YAC1B,iBAAiB,EAAE,MAAM,CAAC;SAC3B,CAAC;KACH,CAAC;CACH"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview Types for RAG audit trails — full transparency on memory/RAG operations.
|
|
3
|
+
* Captures per-operation details (vector search, graph search, reranking, embeddings)
|
|
4
|
+
* with token usage, cost tracking, and source attribution.
|
|
5
|
+
* @module @framers/agentos/rag/audit
|
|
6
|
+
*/
|
|
7
|
+
export {};
|
|
8
|
+
//# sourceMappingURL=RAGAuditTypes.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"RAGAuditTypes.js","sourceRoot":"","sources":["../../../src/rag/audit/RAGAuditTypes.ts"],"names":[],"mappings":"AAAA;;;;;GAKG"}
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview RAG Audit Trail module — types and collector for transparent
|
|
3
|
+
* tracking of RAG operations (vector search, graph search, reranking, embeddings).
|
|
4
|
+
* @module @framers/agentos/rag/audit
|
|
5
|
+
*/
|
|
6
|
+
export type { RAGAuditTrail, RAGOperationEntry, RAGSourceAttribution, } from './RAGAuditTypes.js';
|
|
7
|
+
export { RAGAuditCollector, RAGOperationHandle, type RAGAuditCollectorOptions, } from './RAGAuditCollector.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/rag/audit/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,oBAAoB,GACrB,MAAM,oBAAoB,CAAC;AAE5B,OAAO,EACL,iBAAiB,EACjB,kBAAkB,EAClB,KAAK,wBAAwB,GAC9B,MAAM,wBAAwB,CAAC"}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @fileoverview RAG Audit Trail module — types and collector for transparent
|
|
3
|
+
* tracking of RAG operations (vector search, graph search, reranking, embeddings).
|
|
4
|
+
* @module @framers/agentos/rag/audit
|
|
5
|
+
*/
|
|
6
|
+
export { RAGAuditCollector, RAGOperationHandle, } from './RAGAuditCollector.js';
|
|
7
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/rag/audit/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAQH,OAAO,EACL,iBAAiB,EACjB,kBAAkB,GAEnB,MAAM,wBAAwB,CAAC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"GraphRAGEngine.d.ts","sourceRoot":"","sources":["../../../src/rag/graphrag/GraphRAGEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAkB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAOjE,UAAU,WAAW;IACnB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACvG;AAED,UAAU,kBAAkB;IAC1B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzE,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACtE,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5E;AAuBD,qBAAa,cAAe,YAAW,eAAe;IACpD,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,aAAa,CAAkB;IAGvC,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,mBAAmB,CAA0B;IACrD,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,2BAA2B,CAAmE;IACtG,OAAO,CAAC,iCAAiC,CAAyE;IAGlH,OAAO,CAAC,KAAK,CAAS;IAGtB,OAAO,CAAC,WAAW,CAAC,CAAe;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAAC,CAAc;IAClC,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAEhD,OAAO,CAAC,WAAW,CAAuB;gBAE9B,IAAI,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;QACrC,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC;YASa,yBAAyB;YA0BzB,mBAAmB;IA4B3B,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvD,OAAO,CAAC,iBAAiB;IAenB,eAAe,CACnB,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;KAAE,CAAC,GAC1F,OAAO,CAAC;QACT,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sBAAsB,EAAE,MAAM,CAAC;QAC/B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IAkJI,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC;QACzB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CAAC;YA4DY,+BAA+B;YAa/B,UAAU;IAsFxB,OAAO,CAAC,cAAc;IAsDtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,WAAW;IAsCnB,OAAO,CAAC,iBAAiB;YA6DX,wBAAwB;YAyDxB,iBAAiB;YA6EjB,uBAAuB;YAoHvB,0BAA0B;YAoD1B,kBAAkB;IAiDhC,OAAO,CAAC,eAAe;IAajB,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,kBAAkB,CAAC;
|
|
1
|
+
{"version":3,"file":"GraphRAGEngine.d.ts","sourceRoot":"","sources":["../../../src/rag/graphrag/GraphRAGEngine.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;GAeG;AAMH,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,WAAW,EACX,iBAAiB,EACjB,cAAc,EACd,qBAAqB,EACrB,kBAAkB,EAClB,iBAAiB,EAElB,MAAM,gBAAgB,CAAC;AACxB,OAAO,KAAK,EAAE,YAAY,EAAkB,aAAa,EAAE,MAAM,oBAAoB,CAAC;AACtF,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,yBAAyB,CAAC;AAOjE,UAAU,WAAW;IACnB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE;QAAE,SAAS,CAAC,EAAE,MAAM,CAAC;QAAC,WAAW,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;CACvG;AAED,UAAU,kBAAkB;IAC1B,IAAI,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACpC,GAAG,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC;QAAE,OAAO,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IACzE,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC,CAAC;IACtE,GAAG,CAAC,CAAC,GAAG,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,CAAC,EAAE,GAAG,EAAE,GAAG,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;CAC5E;AAuBD,qBAAa,cAAe,YAAW,eAAe;IACpD,OAAO,CAAC,MAAM,CAAkB;IAChC,OAAO,CAAC,aAAa,CAAkB;IAGvC,OAAO,CAAC,QAAQ,CAAuC;IACvD,OAAO,CAAC,aAAa,CAA6C;IAClE,OAAO,CAAC,WAAW,CAA0C;IAC7D,OAAO,CAAC,mBAAmB,CAA0B;IACrD,OAAO,CAAC,sBAAsB,CAAkC;IAChE,OAAO,CAAC,2BAA2B,CAAmE;IACtG,OAAO,CAAC,iCAAiC,CAAyE;IAGlH,OAAO,CAAC,KAAK,CAAS;IAGtB,OAAO,CAAC,WAAW,CAAC,CAAe;IACnC,OAAO,CAAC,gBAAgB,CAAC,CAAoB;IAC7C,OAAO,CAAC,WAAW,CAAC,CAAc;IAClC,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAEhD,OAAO,CAAC,WAAW,CAAuB;gBAE9B,IAAI,CAAC,EAAE;QACjB,WAAW,CAAC,EAAE,YAAY,CAAC;QAC3B,gBAAgB,CAAC,EAAE,iBAAiB,CAAC;QACrC,WAAW,CAAC,EAAE,WAAW,CAAC;QAC1B,kBAAkB,CAAC,EAAE,kBAAkB,CAAC;KACzC;YASa,yBAAyB;YA0BzB,mBAAmB;IA4B3B,UAAU,CAAC,MAAM,EAAE,cAAc,GAAG,OAAO,CAAC,IAAI,CAAC;IAkDvD,OAAO,CAAC,iBAAiB;IAenB,eAAe,CACnB,SAAS,EAAE,KAAK,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAA;KAAE,CAAC,GAC1F,OAAO,CAAC;QACT,iBAAiB,EAAE,MAAM,CAAC;QAC1B,sBAAsB,EAAE,MAAM,CAAC;QAC/B,mBAAmB,EAAE,MAAM,CAAC;QAC5B,kBAAkB,EAAE,MAAM,CAAC;KAC5B,CAAC;IAkJI,eAAe,CAAC,WAAW,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC;QACpD,gBAAgB,EAAE,MAAM,CAAC;QACzB,mBAAmB,EAAE,MAAM,CAAC;KAC7B,CAAC;YA4DY,+BAA+B;YAa/B,UAAU;IAsFxB,OAAO,CAAC,cAAc;IAsDtB,OAAO,CAAC,sBAAsB;IAQ9B,OAAO,CAAC,gBAAgB;IASxB,OAAO,CAAC,WAAW;IAsCnB,OAAO,CAAC,iBAAiB;YA6DX,wBAAwB;YAyDxB,iBAAiB;YA6EjB,uBAAuB;YAoHvB,0BAA0B;YAoD1B,kBAAkB;IAiDhC,OAAO,CAAC,eAAe;IAajB,YAAY,CAChB,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,kBAAkB,CAAC;IAwKxB,WAAW,CACf,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,qBAAqB,GAC9B,OAAO,CAAC,iBAAiB,CAAC;IAiKvB,WAAW,CAAC,OAAO,CAAC,EAAE;QAAE,IAAI,CAAC,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC;IAShF,gBAAgB,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,iBAAiB,EAAE,CAAC;IAOhE,cAAc,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,OAAO,CAAC,cAAc,EAAE,CAAC;IASzD,QAAQ,IAAI,OAAO,CAAC;QACxB,aAAa,EAAE,MAAM,CAAC;QACtB,kBAAkB,EAAE,MAAM,CAAC;QAC3B,gBAAgB,EAAE,MAAM,CAAC;QACzB,eAAe,EAAE,MAAM,CAAC;QACxB,iBAAiB,EAAE,MAAM,CAAC;KAC3B,CAAC;IAYI,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAsBtB,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;YAcjB,uBAAuB;YAkGvB,mBAAmB;YAkJnB,UAAU;IAyHxB,OAAO,CAAC,yBAAyB;IA4CjC,OAAO,CAAC,+BAA+B;YA0BzB,2CAA2C;YAmD3C,2BAA2B;CA6F1C"}
|
|
@@ -961,6 +961,32 @@ ${summaryContext}
|
|
|
961
961
|
|
|
962
962
|
Provide a comprehensive answer based on the information above.`, { maxTokens: 500, temperature: 0 });
|
|
963
963
|
const synthesisTimeMs = Date.now() - synthesisStart;
|
|
964
|
+
// Audit: record graph global search with LLM synthesis
|
|
965
|
+
if (options?.auditCollector) {
|
|
966
|
+
const auditOp = options.auditCollector.startOperation('graph_global');
|
|
967
|
+
auditOp.setGraphDetails({
|
|
968
|
+
entitiesMatched: 0,
|
|
969
|
+
communitiesSearched: this.communities.size,
|
|
970
|
+
traversalTimeMs: Date.now() - startTime,
|
|
971
|
+
});
|
|
972
|
+
auditOp.addSources(matchedCommunities.map(c => ({
|
|
973
|
+
id: c.communityId,
|
|
974
|
+
originalDocumentId: c.communityId,
|
|
975
|
+
content: c.summary,
|
|
976
|
+
relevanceScore: c.relevanceScore,
|
|
977
|
+
source: `community:${c.title}`,
|
|
978
|
+
})));
|
|
979
|
+
// Estimate LLM tokens for synthesis (~4 chars per token)
|
|
980
|
+
const promptChars = summaryContext.length + query.length + 100;
|
|
981
|
+
const completionChars = answer.length;
|
|
982
|
+
auditOp.setTokenUsage({
|
|
983
|
+
embeddingTokens: 0,
|
|
984
|
+
llmPromptTokens: Math.ceil(promptChars / 4),
|
|
985
|
+
llmCompletionTokens: Math.ceil(completionChars / 4),
|
|
986
|
+
totalTokens: Math.ceil((promptChars + completionChars) / 4),
|
|
987
|
+
});
|
|
988
|
+
auditOp.complete(matchedCommunities.length);
|
|
989
|
+
}
|
|
964
990
|
return {
|
|
965
991
|
query,
|
|
966
992
|
answer,
|
|
@@ -976,6 +1002,23 @@ Provide a comprehensive answer based on the information above.`, { maxTokens: 50
|
|
|
976
1002
|
answer = matchedCommunities
|
|
977
1003
|
.map(c => `${c.title}: ${c.summary}`)
|
|
978
1004
|
.join('\n\n');
|
|
1005
|
+
// Audit: record graph global search
|
|
1006
|
+
if (options?.auditCollector) {
|
|
1007
|
+
const auditOp = options.auditCollector.startOperation('graph_global');
|
|
1008
|
+
auditOp.setGraphDetails({
|
|
1009
|
+
entitiesMatched: 0,
|
|
1010
|
+
communitiesSearched: this.communities.size,
|
|
1011
|
+
traversalTimeMs: Date.now() - startTime,
|
|
1012
|
+
});
|
|
1013
|
+
auditOp.addSources(matchedCommunities.map(c => ({
|
|
1014
|
+
id: c.communityId,
|
|
1015
|
+
originalDocumentId: c.communityId,
|
|
1016
|
+
content: c.summary,
|
|
1017
|
+
relevanceScore: c.relevanceScore,
|
|
1018
|
+
source: `community:${c.title}`,
|
|
1019
|
+
})));
|
|
1020
|
+
auditOp.complete(matchedCommunities.length);
|
|
1021
|
+
}
|
|
979
1022
|
return {
|
|
980
1023
|
query,
|
|
981
1024
|
answer,
|
|
@@ -1089,6 +1132,23 @@ Provide a comprehensive answer based on the information above.`, { maxTokens: 50
|
|
|
1089
1132
|
'## Community Context',
|
|
1090
1133
|
commContext,
|
|
1091
1134
|
].join('\n');
|
|
1135
|
+
// Audit: record graph local search
|
|
1136
|
+
if (options?.auditCollector) {
|
|
1137
|
+
const auditOp = options.auditCollector.startOperation('graph_local');
|
|
1138
|
+
auditOp.setGraphDetails({
|
|
1139
|
+
entitiesMatched: matchedEntities.length,
|
|
1140
|
+
communitiesSearched: seenCommunities.size,
|
|
1141
|
+
traversalTimeMs: graphTraversalTimeMs,
|
|
1142
|
+
});
|
|
1143
|
+
auditOp.addSources(matchedEntities.map(e => ({
|
|
1144
|
+
id: e.id,
|
|
1145
|
+
originalDocumentId: e.sourceDocumentIds[0] ?? e.id,
|
|
1146
|
+
content: `${e.name} (${e.type}): ${e.description}`,
|
|
1147
|
+
relevanceScore: e.relevanceScore,
|
|
1148
|
+
source: `entity:${e.name}`,
|
|
1149
|
+
})));
|
|
1150
|
+
auditOp.complete(matchedEntities.length);
|
|
1151
|
+
}
|
|
1092
1152
|
return {
|
|
1093
1153
|
query,
|
|
1094
1154
|
entities: matchedEntities,
|