@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +847 -6
- package/LICENSE.md +15 -0
- package/docs/INVESTIGATION_WORKFLOW.md +314 -0
- package/package.json +34 -26
- package/scripts/download.ts +22 -1
- package/scripts/generate-wm-templates.ts +2 -1
- package/src/cli.ts +967 -155
- package/src/commands/__tests__/prepare.test.ts +11 -6
- package/src/commands/__tests__/run.test.ts +10 -8
- package/src/commands/clean.ts +147 -0
- package/src/commands/deterministic-ids.ts +126 -0
- package/src/commands/find-prohibited.ts +299 -0
- package/src/commands/investigate.ts +2437 -0
- package/src/commands/list-partial.ts +137 -0
- package/src/commands/obscure-thread-ids.ts +237 -0
- package/src/commands/partial-results.ts +198 -0
- package/src/commands/precompute-embeddings.ts +302 -0
- package/src/commands/prepare.ts +732 -105
- package/src/commands/reconcile.ts +383 -0
- package/src/commands/run.ts +1240 -121
- package/src/commands/sessions.ts +126 -0
- package/src/commands/sync.ts +152 -0
- package/src/commands/test-cleaned.ts +53 -0
- package/src/commands/test-incremental.ts +120 -0
- package/src/commands/test-messages.ts +95 -0
- package/src/commands/test-prohibited.ts +244 -0
- package/src/commands/test-thread.ts +136 -0
- package/src/commands/tokens.ts +625 -0
- package/src/config.ts +1253 -72
- package/src/data/types.ts +48 -6
- package/src/embeddings/cached-openai-embedding-model.ts +0 -1
- package/src/evaluation/longmemeval-metric.ts +186 -103
- package/src/processors/content-sanitizer.ts +153 -0
- package/src/processors/date-injector.ts +111 -0
- package/src/processors/index.ts +3 -0
- package/src/processors/observation-semantic-filter.ts +1142 -0
- package/src/retry-model.ts +5 -3
- package/src/storage/__tests__/benchmark-store.test.ts +18 -35
- package/src/storage/benchmark-store.ts +140 -153
- package/src/storage/benchmark-vector.ts +1 -1
- package/src/storage/index.ts +1 -0
- package/src/storage/persistable-inmemory.ts +133 -0
- package/src/test-utils/mock-model.ts +35 -0
|
@@ -0,0 +1,1142 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* ObservationSemanticFilter - RAG-based filtering for OM observations
|
|
3
|
+
*
|
|
4
|
+
* This processor works alongside ObservationalMemory to filter observations
|
|
5
|
+
* based on semantic similarity to the user's query. It's designed to help
|
|
6
|
+
* models like GPT-4o that struggle with dense context by providing only
|
|
7
|
+
* the most relevant observations.
|
|
8
|
+
*
|
|
9
|
+
* Flow:
|
|
10
|
+
* 1. OM's processInputStep loads observations and stores them in state
|
|
11
|
+
* 2. This processor's processInputStep:
|
|
12
|
+
* - Embeds the user's query
|
|
13
|
+
* - Retrieves relevant observation chunks from vector store
|
|
14
|
+
* - Injects filtered observations into context (replacing OM's full injection)
|
|
15
|
+
*
|
|
16
|
+
* 3. OM's processOutputResult creates new observations
|
|
17
|
+
* 4. This processor's processOutputResult:
|
|
18
|
+
* - Chunks new observations by line
|
|
19
|
+
* - Embeds and stores chunks in vector store
|
|
20
|
+
*/
|
|
21
|
+
|
|
22
|
+
import { embed, embedMany } from 'ai';
|
|
23
|
+
import type { EmbeddingModel } from 'ai';
|
|
24
|
+
import type { Processor, ProcessInputStepArgs, ProcessOutputResultArgs } from '@mastra/core/processors';
|
|
25
|
+
import type { MastraDBMessage } from '@mastra/core/agent';
|
|
26
|
+
import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
|
|
27
|
+
import { join } from 'path';
|
|
28
|
+
import xxhash from 'xxhash-wasm';
|
|
29
|
+
|
|
30
|
+
// ============================================================================
|
|
31
|
+
// Types
|
|
32
|
+
// ============================================================================
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Metadata stored with each observation chunk in the vector store
|
|
36
|
+
*/
|
|
37
|
+
export interface ObservationChunkMetadata {
|
|
38
|
+
/** The thread this observation came from */
|
|
39
|
+
threadId: string;
|
|
40
|
+
/** Resource ID for cross-thread queries */
|
|
41
|
+
resourceId?: string;
|
|
42
|
+
/** The date group header this observation belongs to (e.g., "Date: May 30, 2023") */
|
|
43
|
+
dateGroup: string;
|
|
44
|
+
/** Time extracted from the observation line (e.g., "17:27") */
|
|
45
|
+
time?: string;
|
|
46
|
+
/** Labels extracted from the observation (e.g., [preference], [fact]) */
|
|
47
|
+
labels?: string[];
|
|
48
|
+
/** Index of this line within the observation block */
|
|
49
|
+
lineIndex: number;
|
|
50
|
+
/** The full observation line content */
|
|
51
|
+
content: string;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* A chunk stored in the vector store
|
|
56
|
+
*/
|
|
57
|
+
export interface ObservationChunk {
|
|
58
|
+
id: string;
|
|
59
|
+
embedding: number[];
|
|
60
|
+
metadata: ObservationChunkMetadata;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Configuration for the ObservationSemanticFilter
|
|
65
|
+
*/
|
|
66
|
+
export interface ObservationSemanticFilterConfig {
|
|
67
|
+
/**
|
|
68
|
+
* Embedding model to use for chunking and retrieval
|
|
69
|
+
* @example fastembed.small from @mastra/fastembed
|
|
70
|
+
*/
|
|
71
|
+
embedder: EmbeddingModel<string>;
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* Number of top chunks to retrieve per query
|
|
75
|
+
* @default 50
|
|
76
|
+
*/
|
|
77
|
+
topK?: number;
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* Minimum similarity score (0-1) for a chunk to be included
|
|
81
|
+
* @default 0.3
|
|
82
|
+
*/
|
|
83
|
+
minSimilarity?: number;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Whether to include <current-task> in filtered output
|
|
87
|
+
* @default false (saves tokens for benchmarks)
|
|
88
|
+
*/
|
|
89
|
+
includeCurrentTask?: boolean;
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Whether to include <suggested-response> in filtered output
|
|
93
|
+
* @default false (saves tokens for benchmarks)
|
|
94
|
+
*/
|
|
95
|
+
includeSuggestedResponse?: boolean;
|
|
96
|
+
|
|
97
|
+
/**
|
|
98
|
+
* Whether to include <patterns> in filtered output
|
|
99
|
+
* @default true (patterns are already compressed)
|
|
100
|
+
*/
|
|
101
|
+
includePatterns?: boolean;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Directory to cache embeddings on disk.
|
|
105
|
+
* Embeddings are stored by content hash (xxhash) so they can be reused across configs.
|
|
106
|
+
* @default undefined (no caching)
|
|
107
|
+
*/
|
|
108
|
+
cacheDir?: string;
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Enable score-based context expansion.
|
|
112
|
+
* High-scoring matches will include surrounding observation lines.
|
|
113
|
+
* @default true
|
|
114
|
+
*/
|
|
115
|
+
expandContext?: boolean;
|
|
116
|
+
|
|
117
|
+
/**
|
|
118
|
+
* Similarity threshold for high-confidence matches (±2 lines)
|
|
119
|
+
* @default 0.6
|
|
120
|
+
*/
|
|
121
|
+
highScoreThreshold?: number;
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Similarity threshold for medium-confidence matches (±1 line)
|
|
125
|
+
* @default 0.4
|
|
126
|
+
*/
|
|
127
|
+
mediumScoreThreshold?: number;
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Enable preference boost queries.
|
|
131
|
+
* Runs additional searches for "user prefers", "user preference", etc.
|
|
132
|
+
* to ensure user preferences are always included in context.
|
|
133
|
+
* @default true
|
|
134
|
+
*/
|
|
135
|
+
preferenceBoost?: boolean;
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* TopK for preference boost queries
|
|
139
|
+
* @default 50
|
|
140
|
+
*/
|
|
141
|
+
preferenceBoostTopK?: number;
|
|
142
|
+
|
|
143
|
+
/**
|
|
144
|
+
* Minimum similarity for preference boost queries (lower than main query)
|
|
145
|
+
* @default 0.35
|
|
146
|
+
*/
|
|
147
|
+
preferenceBoostMinSimilarity?: number;
|
|
148
|
+
|
|
149
|
+
/**
|
|
150
|
+
* Enable query expansion for complex/multi-part questions.
|
|
151
|
+
* Extracts quoted text, sentences, and key phrases as separate queries.
|
|
152
|
+
* @default true
|
|
153
|
+
*/
|
|
154
|
+
expandQueries?: boolean;
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* TopK for each expanded query (smaller than main query since results are merged)
|
|
158
|
+
* @default 20
|
|
159
|
+
*/
|
|
160
|
+
queryExpansionTopK?: number;
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/**
|
|
164
|
+
* In-memory vector store for observation chunks
|
|
165
|
+
* Simple implementation for experimentation - can be replaced with LibSQL later
|
|
166
|
+
*/
|
|
167
|
+
export class InMemoryVectorStore {
|
|
168
|
+
private chunks: Map<string, ObservationChunk> = new Map();
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Add or update chunks in the store
|
|
172
|
+
*/
|
|
173
|
+
upsert(chunks: ObservationChunk[]): void {
|
|
174
|
+
for (const chunk of chunks) {
|
|
175
|
+
this.chunks.set(chunk.id, chunk);
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/**
|
|
180
|
+
* Query chunks by similarity to a query embedding
|
|
181
|
+
*/
|
|
182
|
+
query(
|
|
183
|
+
queryEmbedding: number[],
|
|
184
|
+
options: { topK: number; minSimilarity: number; resourceId?: string },
|
|
185
|
+
): Array<{ chunk: ObservationChunk; similarity: number }> {
|
|
186
|
+
const results: Array<{ chunk: ObservationChunk; similarity: number }> = [];
|
|
187
|
+
|
|
188
|
+
for (const chunk of Array.from(this.chunks.values())) {
|
|
189
|
+
// Filter by resourceId if provided
|
|
190
|
+
if (options.resourceId && chunk.metadata.resourceId !== options.resourceId) {
|
|
191
|
+
continue;
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
const similarity = this.cosineSimilarity(queryEmbedding, chunk.embedding);
|
|
195
|
+
if (similarity >= options.minSimilarity) {
|
|
196
|
+
results.push({ chunk, similarity });
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
|
|
200
|
+
// Sort by similarity descending and take topK
|
|
201
|
+
return results.sort((a, b) => b.similarity - a.similarity).slice(0, options.topK);
|
|
202
|
+
}
|
|
203
|
+
|
|
204
|
+
/**
|
|
205
|
+
* Get all chunks (for debugging/inspection)
|
|
206
|
+
*/
|
|
207
|
+
getAll(): ObservationChunk[] {
|
|
208
|
+
return Array.from(this.chunks.values());
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Clear all chunks
|
|
213
|
+
*/
|
|
214
|
+
clear(): void {
|
|
215
|
+
this.chunks.clear();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Get chunk count
|
|
220
|
+
*/
|
|
221
|
+
get size(): number {
|
|
222
|
+
return this.chunks.size;
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
/**
|
|
226
|
+
* Cosine similarity between two vectors
|
|
227
|
+
*/
|
|
228
|
+
private cosineSimilarity(a: number[], b: number[]): number {
|
|
229
|
+
if (a.length !== b.length) return 0;
|
|
230
|
+
|
|
231
|
+
let dotProduct = 0;
|
|
232
|
+
let normA = 0;
|
|
233
|
+
let normB = 0;
|
|
234
|
+
|
|
235
|
+
for (let i = 0; i < a.length; i++) {
|
|
236
|
+
dotProduct += a[i] * b[i];
|
|
237
|
+
normA += a[i] * a[i];
|
|
238
|
+
normB += b[i] * b[i];
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
|
|
242
|
+
return magnitude === 0 ? 0 : dotProduct / magnitude;
|
|
243
|
+
}
|
|
244
|
+
|
|
245
|
+
/**
|
|
246
|
+
* Serialize store to JSON (for persistence)
|
|
247
|
+
*/
|
|
248
|
+
toJSON(): { chunks: ObservationChunk[] } {
|
|
249
|
+
return { chunks: Array.from(this.chunks.values()) };
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
/**
|
|
253
|
+
* Load store from JSON
|
|
254
|
+
*/
|
|
255
|
+
static fromJSON(data: { chunks: ObservationChunk[] }): InMemoryVectorStore {
|
|
256
|
+
const store = new InMemoryVectorStore();
|
|
257
|
+
store.upsert(data.chunks);
|
|
258
|
+
return store;
|
|
259
|
+
}
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
// ============================================================================
|
|
263
|
+
// Parsing Helpers
|
|
264
|
+
// ============================================================================
|
|
265
|
+
|
|
266
|
+
/**
|
|
267
|
+
* Parsed observation with full context (thread, date group, content)
|
|
268
|
+
*/
|
|
269
|
+
export interface ParsedObservation {
|
|
270
|
+
line: string;
|
|
271
|
+
lineIndex: number;
|
|
272
|
+
threadId: string;
|
|
273
|
+
dateGroup: string; // The date header (e.g., "Date: May 30, 2023")
|
|
274
|
+
time?: string; // Time extracted from line (e.g., "17:27")
|
|
275
|
+
labels?: string[];
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
/**
|
|
279
|
+
* Parse observation lines from the <observations> block, preserving thread and date context
|
|
280
|
+
* Each line is one observation item (emoji + time + content + labels)
|
|
281
|
+
*/
|
|
282
|
+
export function parseObservationLines(observations: string): ParsedObservation[] {
|
|
283
|
+
const lines = observations.split('\n');
|
|
284
|
+
const results: ParsedObservation[] = [];
|
|
285
|
+
|
|
286
|
+
let currentThreadId = 'default';
|
|
287
|
+
let currentDateGroup = 'Unknown Date';
|
|
288
|
+
|
|
289
|
+
for (let i = 0; i < lines.length; i++) {
|
|
290
|
+
const line = lines[i].trim();
|
|
291
|
+
if (!line) continue;
|
|
292
|
+
|
|
293
|
+
// Check for thread headers
|
|
294
|
+
const threadMatch = line.match(/^<thread\s+id="([^"]+)"/i) || line.match(/^<other-conversation\s+id="([^"]+)"/i);
|
|
295
|
+
if (threadMatch) {
|
|
296
|
+
currentThreadId = threadMatch[1];
|
|
297
|
+
continue;
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
// Check for thread closing tags
|
|
301
|
+
if (line.match(/^<\/thread>/i) || line.match(/^<\/other-conversation>/i)) {
|
|
302
|
+
currentThreadId = 'default';
|
|
303
|
+
continue;
|
|
304
|
+
}
|
|
305
|
+
|
|
306
|
+
// Check for date headers
|
|
307
|
+
// Format 1: "## 2023-05-15" or "# 2023-05-15"
|
|
308
|
+
const dateHeaderMatch1 = line.match(/^##?\s*(\d{4}-\d{2}-\d{2})/);
|
|
309
|
+
if (dateHeaderMatch1) {
|
|
310
|
+
currentDateGroup = `Date: ${dateHeaderMatch1[1]}`;
|
|
311
|
+
continue;
|
|
312
|
+
}
|
|
313
|
+
// Format 2: "Date: May 30, 2023" or "Date: Dec 20, 2022"
|
|
314
|
+
const dateHeaderMatch2 = line.match(/^(Date:\s+.+)$/i);
|
|
315
|
+
if (dateHeaderMatch2) {
|
|
316
|
+
currentDateGroup = dateHeaderMatch2[1];
|
|
317
|
+
continue;
|
|
318
|
+
}
|
|
319
|
+
|
|
320
|
+
// Skip other XML tags
|
|
321
|
+
if (line.startsWith('<') || line.startsWith('</')) {
|
|
322
|
+
continue;
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
// This is an observation line - extract time and labels
|
|
326
|
+
// Time format: (HH:MM) at the start after emoji
|
|
327
|
+
const timeMatch = line.match(/\((\d{2}:\d{2})\)/);
|
|
328
|
+
const time = timeMatch ? timeMatch[1] : undefined;
|
|
329
|
+
|
|
330
|
+
// Extract labels (format: [label])
|
|
331
|
+
const labelMatches = line.match(/\[([^\]]+)\]/g);
|
|
332
|
+
const labels = labelMatches?.map(l => l.slice(1, -1));
|
|
333
|
+
|
|
334
|
+
results.push({
|
|
335
|
+
line,
|
|
336
|
+
lineIndex: i,
|
|
337
|
+
threadId: currentThreadId,
|
|
338
|
+
dateGroup: currentDateGroup,
|
|
339
|
+
time,
|
|
340
|
+
labels,
|
|
341
|
+
});
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
return results;
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
/**
|
|
348
|
+
* Rebuild observations with thread and date structure from retrieved chunks
|
|
349
|
+
*/
|
|
350
|
+
export function rebuildObservationsWithStructure(
|
|
351
|
+
chunks: Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }>,
|
|
352
|
+
): string {
|
|
353
|
+
// Group by thread, then by date
|
|
354
|
+
const threadGroups = new Map<string, Map<string, string[]>>();
|
|
355
|
+
|
|
356
|
+
for (const chunk of chunks) {
|
|
357
|
+
const { threadId, dateGroup, content } = chunk.metadata;
|
|
358
|
+
|
|
359
|
+
if (!threadGroups.has(threadId)) {
|
|
360
|
+
threadGroups.set(threadId, new Map());
|
|
361
|
+
}
|
|
362
|
+
const dateGroups = threadGroups.get(threadId)!;
|
|
363
|
+
|
|
364
|
+
if (!dateGroups.has(dateGroup)) {
|
|
365
|
+
dateGroups.set(dateGroup, []);
|
|
366
|
+
}
|
|
367
|
+
dateGroups.get(dateGroup)!.push(content);
|
|
368
|
+
}
|
|
369
|
+
|
|
370
|
+
// Build output with structure
|
|
371
|
+
const parts: string[] = [];
|
|
372
|
+
|
|
373
|
+
for (const [threadId, dateGroups] of Array.from(threadGroups.entries())) {
|
|
374
|
+
// Add thread wrapper if not default
|
|
375
|
+
if (threadId !== 'default') {
|
|
376
|
+
parts.push(`<thread id="${threadId}">`);
|
|
377
|
+
}
|
|
378
|
+
|
|
379
|
+
for (const [dateGroup, observations] of Array.from(dateGroups.entries())) {
|
|
380
|
+
parts.push(dateGroup);
|
|
381
|
+
parts.push(...observations);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
if (threadId !== 'default') {
|
|
385
|
+
parts.push('</thread>');
|
|
386
|
+
}
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
return parts.join('\n');
|
|
390
|
+
}
|
|
391
|
+
|
|
392
|
+
/**
|
|
393
|
+
* Extract the <observations> content from formatted context
|
|
394
|
+
*/
|
|
395
|
+
export function extractObservationsBlock(formattedContext: string): string | null {
|
|
396
|
+
const match = formattedContext.match(/<observations>([\s\S]*?)<\/observations>/);
|
|
397
|
+
return match ? match[1].trim() : null;
|
|
398
|
+
}
|
|
399
|
+
|
|
400
|
+
/**
|
|
401
|
+
* Extract the <patterns> content from formatted context
|
|
402
|
+
*/
|
|
403
|
+
export function extractPatternsBlock(formattedContext: string): string | null {
|
|
404
|
+
const match = formattedContext.match(/<patterns>([\s\S]*?)<\/patterns>/);
|
|
405
|
+
return match ? match[1].trim() : null;
|
|
406
|
+
}
|
|
407
|
+
|
|
408
|
+
/**
|
|
409
|
+
* Extract the <current-task> content from formatted context
|
|
410
|
+
*/
|
|
411
|
+
export function extractCurrentTaskBlock(formattedContext: string): string | null {
|
|
412
|
+
const match = formattedContext.match(/<current-task>([\s\S]*?)<\/current-task>/);
|
|
413
|
+
return match ? match[1].trim() : null;
|
|
414
|
+
}
|
|
415
|
+
|
|
416
|
+
/**
|
|
417
|
+
* Extract the <suggested-response> content from formatted context
|
|
418
|
+
*/
|
|
419
|
+
export function extractSuggestedResponseBlock(formattedContext: string): string | null {
|
|
420
|
+
const match = formattedContext.match(/<suggested-response>([\s\S]*?)<\/suggested-response>/);
|
|
421
|
+
return match ? match[1].trim() : null;
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Get the most recent user message from the message list
|
|
426
|
+
*/
|
|
427
|
+
export function getLatestUserMessage(messages: MastraDBMessage[]): string | null {
|
|
428
|
+
for (let i = messages.length - 1; i >= 0; i--) {
|
|
429
|
+
const msg = messages[i];
|
|
430
|
+
if (msg.role === 'user') {
|
|
431
|
+
const content = msg.content;
|
|
432
|
+
|
|
433
|
+
// Handle string content (legacy format)
|
|
434
|
+
if (typeof content === 'string') {
|
|
435
|
+
return content;
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Handle MastraMessageContentV2 format (format: 2 with parts array)
|
|
439
|
+
if (content && typeof content === 'object' && 'format' in content && content.format === 2) {
|
|
440
|
+
// First check if there's a direct content string
|
|
441
|
+
if (typeof content.content === 'string' && content.content) {
|
|
442
|
+
return content.content;
|
|
443
|
+
}
|
|
444
|
+
// Otherwise extract from parts
|
|
445
|
+
if (Array.isArray(content.parts)) {
|
|
446
|
+
const textParts = content.parts
|
|
447
|
+
.filter(
|
|
448
|
+
(p): p is { type: 'text'; text: string } =>
|
|
449
|
+
p && typeof p === 'object' && p.type === 'text' && typeof p.text === 'string',
|
|
450
|
+
)
|
|
451
|
+
.map(p => p.text);
|
|
452
|
+
if (textParts.length > 0) {
|
|
453
|
+
return textParts.join(' ');
|
|
454
|
+
}
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
// Handle plain array content (older format)
|
|
459
|
+
if (Array.isArray(content)) {
|
|
460
|
+
const textParts = content
|
|
461
|
+
.filter(
|
|
462
|
+
(p): p is { type: 'text'; text: string } =>
|
|
463
|
+
p && typeof p === 'object' && p.type === 'text' && typeof p.text === 'string',
|
|
464
|
+
)
|
|
465
|
+
.map(p => p.text);
|
|
466
|
+
return textParts.join(' ') || null;
|
|
467
|
+
}
|
|
468
|
+
}
|
|
469
|
+
}
|
|
470
|
+
return null;
|
|
471
|
+
}
|
|
472
|
+
|
|
473
|
+
// ============================================================================
|
|
474
|
+
// Processor Implementation
|
|
475
|
+
// ============================================================================
|
|
476
|
+
|
|
477
|
+
/**
|
|
478
|
+
* ObservationSemanticFilter processor
|
|
479
|
+
*
|
|
480
|
+
* Works with ObservationalMemory to filter observations based on semantic
|
|
481
|
+
* similarity to the user's query.
|
|
482
|
+
*/
|
|
483
|
+
export class ObservationSemanticFilter implements Processor<'observation-semantic-filter'> {
|
|
484
|
+
readonly id = 'observation-semantic-filter' as const;
|
|
485
|
+
readonly name = 'Observation Semantic Filter';
|
|
486
|
+
|
|
487
|
+
private embedder: EmbeddingModel<string>;
|
|
488
|
+
private vectorStore: InMemoryVectorStore;
|
|
489
|
+
private topK: number;
|
|
490
|
+
private minSimilarity: number;
|
|
491
|
+
private includeCurrentTask: boolean;
|
|
492
|
+
private includeSuggestedResponse: boolean;
|
|
493
|
+
private includePatterns: boolean;
|
|
494
|
+
private cacheDir?: string;
|
|
495
|
+
private hasher = xxhash();
|
|
496
|
+
private embeddingCache = new Map<string, number[]>(); // hash -> embedding
|
|
497
|
+
|
|
498
|
+
// Context expansion settings
|
|
499
|
+
private expandContext: boolean;
|
|
500
|
+
private highScoreThreshold: number;
|
|
501
|
+
private mediumScoreThreshold: number;
|
|
502
|
+
|
|
503
|
+
// Preference boost settings
|
|
504
|
+
private preferenceBoost: boolean;
|
|
505
|
+
private preferenceBoostTopK: number;
|
|
506
|
+
private preferenceBoostMinSimilarity: number;
|
|
507
|
+
|
|
508
|
+
// Store all parsed observations for neighbor lookup during expansion
|
|
509
|
+
private allParsedObservations: ParsedObservation[] = [];
|
|
510
|
+
|
|
511
|
+
// Query expansion settings
|
|
512
|
+
private expandQueries: boolean;
|
|
513
|
+
private queryExpansionTopK: number;
|
|
514
|
+
|
|
515
|
+
constructor(config: ObservationSemanticFilterConfig) {
|
|
516
|
+
this.embedder = config.embedder;
|
|
517
|
+
this.vectorStore = new InMemoryVectorStore();
|
|
518
|
+
this.topK = config.topK ?? 30;
|
|
519
|
+
this.minSimilarity = config.minSimilarity ?? 0.4;
|
|
520
|
+
this.includeCurrentTask = config.includeCurrentTask ?? false;
|
|
521
|
+
this.includeSuggestedResponse = config.includeSuggestedResponse ?? false;
|
|
522
|
+
this.includePatterns = config.includePatterns ?? false;
|
|
523
|
+
this.cacheDir = config.cacheDir;
|
|
524
|
+
this.expandContext = config.expandContext ?? true;
|
|
525
|
+
this.highScoreThreshold = config.highScoreThreshold ?? 0.8;
|
|
526
|
+
this.mediumScoreThreshold = config.mediumScoreThreshold ?? 0.4;
|
|
527
|
+
this.preferenceBoost = config.preferenceBoost ?? true;
|
|
528
|
+
this.preferenceBoostTopK = config.preferenceBoostTopK ?? 10;
|
|
529
|
+
this.preferenceBoostMinSimilarity = config.preferenceBoostMinSimilarity ?? 0.8;
|
|
530
|
+
this.expandQueries = config.expandQueries ?? false;
|
|
531
|
+
this.queryExpansionTopK = config.queryExpansionTopK ?? 5;
|
|
532
|
+
|
|
533
|
+
// Ensure cache directory exists
|
|
534
|
+
if (this.cacheDir && !existsSync(this.cacheDir)) {
|
|
535
|
+
mkdirSync(this.cacheDir, { recursive: true });
|
|
536
|
+
}
|
|
537
|
+
}
|
|
538
|
+
|
|
539
|
+
/**
|
|
540
|
+
* Hash content to create a cache key using xxhash
|
|
541
|
+
*/
|
|
542
|
+
private async hashContent(content: string): Promise<string> {
|
|
543
|
+
const hasher = await this.hasher;
|
|
544
|
+
return hasher.h32ToString(content);
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
/**
|
|
548
|
+
* Get cached embedding for content, or null if not cached
|
|
549
|
+
*/
|
|
550
|
+
private getCachedEmbedding(hash: string): number[] | null {
|
|
551
|
+
// Check in-memory cache first
|
|
552
|
+
if (this.embeddingCache.has(hash)) {
|
|
553
|
+
return this.embeddingCache.get(hash)!;
|
|
554
|
+
}
|
|
555
|
+
|
|
556
|
+
// Check disk cache
|
|
557
|
+
if (this.cacheDir) {
|
|
558
|
+
const cachePath = join(this.cacheDir, `${hash}.json`);
|
|
559
|
+
if (existsSync(cachePath)) {
|
|
560
|
+
try {
|
|
561
|
+
const embedding = JSON.parse(readFileSync(cachePath, 'utf-8'));
|
|
562
|
+
this.embeddingCache.set(hash, embedding); // Store in memory for future lookups
|
|
563
|
+
return embedding;
|
|
564
|
+
} catch {
|
|
565
|
+
// Ignore corrupted cache files
|
|
566
|
+
}
|
|
567
|
+
}
|
|
568
|
+
}
|
|
569
|
+
|
|
570
|
+
return null;
|
|
571
|
+
}
|
|
572
|
+
|
|
573
|
+
/**
|
|
574
|
+
* Save embedding to cache (both in-memory and disk)
|
|
575
|
+
*/
|
|
576
|
+
private saveEmbeddingToCache(hash: string, embedding: number[]): void {
|
|
577
|
+
this.embeddingCache.set(hash, embedding);
|
|
578
|
+
|
|
579
|
+
if (this.cacheDir) {
|
|
580
|
+
const cachePath = join(this.cacheDir, `${hash}.json`);
|
|
581
|
+
try {
|
|
582
|
+
writeFileSync(cachePath, JSON.stringify(embedding));
|
|
583
|
+
} catch {
|
|
584
|
+
// Ignore write errors
|
|
585
|
+
}
|
|
586
|
+
}
|
|
587
|
+
}
|
|
588
|
+
|
|
589
|
+
/**
|
|
590
|
+
* Get the vector store (for external access/persistence)
|
|
591
|
+
*/
|
|
592
|
+
getVectorStore(): InMemoryVectorStore {
|
|
593
|
+
return this.vectorStore;
|
|
594
|
+
}
|
|
595
|
+
|
|
596
|
+
/**
|
|
597
|
+
* Set the vector store (for loading persisted data)
|
|
598
|
+
*/
|
|
599
|
+
setVectorStore(store: InMemoryVectorStore): void {
|
|
600
|
+
this.vectorStore = store;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
/**
|
|
604
|
+
* Expand a complex query into multiple sub-queries for better retrieval.
|
|
605
|
+
* Extracts:
|
|
606
|
+
* - Quoted text (e.g., "The Mountain Meditation")
|
|
607
|
+
* - Individual sentences
|
|
608
|
+
* - Key noun phrases
|
|
609
|
+
*
|
|
610
|
+
* @returns Array of query strings to embed and search
|
|
611
|
+
*/
|
|
612
|
+
private expandQuery(userQuery: string): string[] {
|
|
613
|
+
const queries: string[] = [];
|
|
614
|
+
const seen = new Set<string>();
|
|
615
|
+
|
|
616
|
+
const addQuery = (q: string, clean = true) => {
|
|
617
|
+
// Clean up the query - remove trailing/leading punctuation and quotes
|
|
618
|
+
const cleaned = clean
|
|
619
|
+
? q
|
|
620
|
+
.trim()
|
|
621
|
+
.replace(/^['".,]+|['".,]+$/g, '')
|
|
622
|
+
.trim()
|
|
623
|
+
: q;
|
|
624
|
+
const normalized = cleaned.toLowerCase();
|
|
625
|
+
// Skip very short queries (less than 10 chars) or duplicates
|
|
626
|
+
if (normalized.length >= 10 && !seen.has(normalized)) {
|
|
627
|
+
seen.add(normalized);
|
|
628
|
+
queries.push(cleaned);
|
|
629
|
+
}
|
|
630
|
+
};
|
|
631
|
+
|
|
632
|
+
// 1. Always include the full query (for overall context)
|
|
633
|
+
addQuery(userQuery, false);
|
|
634
|
+
|
|
635
|
+
// 2. Extract quoted text (single or double quotes)
|
|
636
|
+
const quotedMatches = userQuery.match(/["']([^"']+)["']/g);
|
|
637
|
+
if (quotedMatches) {
|
|
638
|
+
for (const match of quotedMatches) {
|
|
639
|
+
// Remove the quotes
|
|
640
|
+
const content = match.slice(1, -1);
|
|
641
|
+
addQuery(content);
|
|
642
|
+
}
|
|
643
|
+
}
|
|
644
|
+
|
|
645
|
+
// 3. Split by sentences and add each meaningful sentence
|
|
646
|
+
// Split on . ! ? but keep the delimiter for context
|
|
647
|
+
const sentences = userQuery.split(/(?<=[.!?])\s+/);
|
|
648
|
+
for (const sentence of sentences) {
|
|
649
|
+
const trimmed = sentence.trim();
|
|
650
|
+
// Only add if it's a substantial sentence (not just "Thanks!" etc.)
|
|
651
|
+
if (trimmed.length >= 20) {
|
|
652
|
+
addQuery(trimmed);
|
|
653
|
+
}
|
|
654
|
+
}
|
|
655
|
+
|
|
656
|
+
// 4. Extract key noun phrases using simple heuristics
|
|
657
|
+
// Look for patterns like "the X", "my X", "a X" where X is a noun phrase
|
|
658
|
+
const nounPhrasePatterns = [
|
|
659
|
+
/(?:the|my|a|an|your)\s+([a-z]+(?:\s+[a-z]+){0,3})/gi,
|
|
660
|
+
/(?:about|for|with|from)\s+([a-z]+(?:\s+[a-z]+){0,3})/gi,
|
|
661
|
+
];
|
|
662
|
+
|
|
663
|
+
for (const pattern of nounPhrasePatterns) {
|
|
664
|
+
let match;
|
|
665
|
+
while ((match = pattern.exec(userQuery)) !== null) {
|
|
666
|
+
const phrase = match[1];
|
|
667
|
+
// Only add substantial phrases
|
|
668
|
+
if (phrase && phrase.length >= 5) {
|
|
669
|
+
addQuery(phrase);
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
674
|
+
// 5. Extract specific terms that might be important
|
|
675
|
+
// Look for capitalized words (proper nouns) that aren't at sentence start
|
|
676
|
+
const words = userQuery.split(/\s+/);
|
|
677
|
+
const properNouns: string[] = [];
|
|
678
|
+
for (let i = 1; i < words.length; i++) {
|
|
679
|
+
const word = words[i];
|
|
680
|
+
// Strip leading quotes for the capital check
|
|
681
|
+
const cleanWord = word.replace(/^['"]/, '');
|
|
682
|
+
// Check if word starts with capital and isn't after sentence-ending punctuation
|
|
683
|
+
if (cleanWord.length > 2 && /^[A-Z]/.test(cleanWord) && !/[.!?]$/.test(words[i - 1] || '')) {
|
|
684
|
+
// Collect consecutive capitalized words, stripping punctuation and quotes
|
|
685
|
+
const phrase = [cleanWord.replace(/[.,!?'"]+$/g, '')];
|
|
686
|
+
let j = i + 1;
|
|
687
|
+
while (j < words.length && /^['"]?[A-Z]/.test(words[j])) {
|
|
688
|
+
phrase.push(words[j].replace(/^['"]/, '').replace(/[.,!?'"]+$/g, ''));
|
|
689
|
+
j++;
|
|
690
|
+
}
|
|
691
|
+
if (phrase.join(' ').length >= 5) {
|
|
692
|
+
properNouns.push(phrase.join(' '));
|
|
693
|
+
}
|
|
694
|
+
}
|
|
695
|
+
}
|
|
696
|
+
for (const noun of properNouns) {
|
|
697
|
+
addQuery(noun);
|
|
698
|
+
}
|
|
699
|
+
|
|
700
|
+
console.log('[RAG Filter] Expanded query into', queries.length, 'sub-queries:', queries.slice(0, 5));
|
|
701
|
+
|
|
702
|
+
return queries;
|
|
703
|
+
}
|
|
704
|
+
|
|
705
|
+
/**
|
|
706
|
+
* Process input at each step - filter observations based on query similarity
|
|
707
|
+
*/
|
|
708
|
+
async processInputStep(args: ProcessInputStepArgs) {
|
|
709
|
+
const { messages, messageList, state } = args;
|
|
710
|
+
|
|
711
|
+
console.log('[RAG Filter] processInputStep called, stepNumber:', args.stepNumber);
|
|
712
|
+
|
|
713
|
+
// Only process on step 0 (initial input)
|
|
714
|
+
// OM has already injected its full observations by this point
|
|
715
|
+
if (args.stepNumber !== 0) {
|
|
716
|
+
return messageList;
|
|
717
|
+
}
|
|
718
|
+
|
|
719
|
+
// Get OM's system messages directly from messageList
|
|
720
|
+
const omSystemMessages = messageList.getSystemMessages('observational-memory');
|
|
721
|
+
console.log('[RAG Filter] Found OM system messages:', omSystemMessages.length);
|
|
722
|
+
if (omSystemMessages.length === 0) {
|
|
723
|
+
// No observations to filter
|
|
724
|
+
return messageList;
|
|
725
|
+
}
|
|
726
|
+
|
|
727
|
+
const omSystemMessage = omSystemMessages[0];
|
|
728
|
+
const fullContext = typeof omSystemMessage.content === 'string' ? omSystemMessage.content : '';
|
|
729
|
+
console.log('[RAG Filter] Full context length:', fullContext.length);
|
|
730
|
+
|
|
731
|
+
// Extract the observations block
|
|
732
|
+
const observationsContent = extractObservationsBlock(fullContext);
|
|
733
|
+
console.log('[RAG Filter] Observations content length:', observationsContent?.length ?? 0);
|
|
734
|
+
if (!observationsContent) {
|
|
735
|
+
console.log('[RAG Filter] No observations block found, returning early');
|
|
736
|
+
return messageList;
|
|
737
|
+
}
|
|
738
|
+
|
|
739
|
+
// Get the user's query for similarity matching
|
|
740
|
+
// Debug: log all messages to understand the structure
|
|
741
|
+
console.log(
|
|
742
|
+
'[RAG Filter] Messages:',
|
|
743
|
+
messages.map(m => {
|
|
744
|
+
const content = m.content as unknown;
|
|
745
|
+
let preview = 'unknown';
|
|
746
|
+
if (typeof content === 'string') {
|
|
747
|
+
preview = (content as string).slice(0, 50);
|
|
748
|
+
} else if (Array.isArray(content)) {
|
|
749
|
+
preview = `array[${(content as unknown[]).length}]`;
|
|
750
|
+
}
|
|
751
|
+
return { role: m.role, contentType: typeof content, contentPreview: preview };
|
|
752
|
+
}),
|
|
753
|
+
);
|
|
754
|
+
|
|
755
|
+
const userQuery = getLatestUserMessage(messages);
|
|
756
|
+
console.log('[RAG Filter] User query found:', !!userQuery, 'messages count:', messages.length);
|
|
757
|
+
if (!userQuery) {
|
|
758
|
+
// No query to match against - keep full observations
|
|
759
|
+
console.log('[RAG Filter] No user query, returning early');
|
|
760
|
+
return messageList;
|
|
761
|
+
}
|
|
762
|
+
|
|
763
|
+
console.log('[RAG Filter] User query:', userQuery?.slice(0, 100));
|
|
764
|
+
|
|
765
|
+
// Index observations (will use cache for already-embedded content)
|
|
766
|
+
if (this.vectorStore.size === 0) {
|
|
767
|
+
console.log('[RAG Filter] Indexing observations...');
|
|
768
|
+
await this.indexObservations(observationsContent, state);
|
|
769
|
+
console.log('[RAG Filter] Indexed', this.vectorStore.size, 'chunks');
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
// Track all results and seen chunk IDs for deduplication
|
|
773
|
+
const seenChunkIds = new Set<string>();
|
|
774
|
+
let allResults: Array<{ chunk: ObservationChunk; similarity: number }> = [];
|
|
775
|
+
let mainQueryResultCount = 0;
|
|
776
|
+
let topSimilarity = 0;
|
|
777
|
+
|
|
778
|
+
// Determine queries to run
|
|
779
|
+
const queriesToRun = this.expandQueries ? this.expandQuery(userQuery) : [userQuery];
|
|
780
|
+
|
|
781
|
+
// Run each query and merge results
|
|
782
|
+
for (let i = 0; i < queriesToRun.length; i++) {
|
|
783
|
+
const query = queriesToRun[i];
|
|
784
|
+
const isMainQuery = i === 0;
|
|
785
|
+
|
|
786
|
+
const { embedding: queryEmbedding } = await embed({
|
|
787
|
+
model: this.embedder,
|
|
788
|
+
value: query,
|
|
789
|
+
});
|
|
790
|
+
|
|
791
|
+
// Use full topK for main query, smaller for expanded queries
|
|
792
|
+
const topK = isMainQuery ? this.topK : this.queryExpansionTopK;
|
|
793
|
+
|
|
794
|
+
const results = this.vectorStore.query(queryEmbedding, {
|
|
795
|
+
topK,
|
|
796
|
+
minSimilarity: isMainQuery ? this.minSimilarity : 0.8,
|
|
797
|
+
});
|
|
798
|
+
|
|
799
|
+
if (isMainQuery) {
|
|
800
|
+
mainQueryResultCount = results.length;
|
|
801
|
+
topSimilarity = results[0]?.similarity ?? 0;
|
|
802
|
+
console.log('[RAG Filter] Main query found', results.length, 'chunks');
|
|
803
|
+
console.log(
|
|
804
|
+
'[RAG Filter] Top 3 similarities:',
|
|
805
|
+
results
|
|
806
|
+
.slice(0, 3)
|
|
807
|
+
.map(r => r.similarity.toFixed(3))
|
|
808
|
+
.join(', '),
|
|
809
|
+
);
|
|
810
|
+
}
|
|
811
|
+
|
|
812
|
+
// Add unique results, keeping highest similarity for duplicates
|
|
813
|
+
for (const result of results) {
|
|
814
|
+
if (!seenChunkIds.has(result.chunk.id)) {
|
|
815
|
+
seenChunkIds.add(result.chunk.id);
|
|
816
|
+
allResults.push(result);
|
|
817
|
+
} else {
|
|
818
|
+
// Update similarity if this query found a higher score
|
|
819
|
+
const existing = allResults.find(r => r.chunk.id === result.chunk.id);
|
|
820
|
+
if (existing && result.similarity > existing.similarity) {
|
|
821
|
+
existing.similarity = result.similarity;
|
|
822
|
+
}
|
|
823
|
+
}
|
|
824
|
+
}
|
|
825
|
+
}
|
|
826
|
+
|
|
827
|
+
if (this.expandQueries && queriesToRun.length > 1) {
|
|
828
|
+
console.log(
|
|
829
|
+
'[RAG Filter] Query expansion added',
|
|
830
|
+
allResults.length - this.topK,
|
|
831
|
+
'additional unique chunks from',
|
|
832
|
+
queriesToRun.length - 1,
|
|
833
|
+
'sub-queries',
|
|
834
|
+
);
|
|
835
|
+
}
|
|
836
|
+
// Preference boost: run additional queries for user preferences
|
|
837
|
+
// This ensures critical user preferences are always included in context
|
|
838
|
+
if (this.preferenceBoost) {
|
|
839
|
+
const preferenceQueries = ['likes, prefers, wants, favourites'];
|
|
840
|
+
// const preferenceQueries = ['user prefers', 'user preference', 'user likes', 'user wants', "user's favorite"];
|
|
841
|
+
let preferenceHits = 0;
|
|
842
|
+
|
|
843
|
+
for (const prefQuery of preferenceQueries) {
|
|
844
|
+
const { embedding: prefEmbedding } = await embed({
|
|
845
|
+
model: this.embedder,
|
|
846
|
+
value: prefQuery,
|
|
847
|
+
});
|
|
848
|
+
|
|
849
|
+
const prefResults = this.vectorStore.query(prefEmbedding, {
|
|
850
|
+
topK: this.preferenceBoostTopK,
|
|
851
|
+
minSimilarity: this.preferenceBoostMinSimilarity,
|
|
852
|
+
});
|
|
853
|
+
|
|
854
|
+
// Add unique results (not already in main results or expanded queries)
|
|
855
|
+
for (const result of prefResults) {
|
|
856
|
+
if (!seenChunkIds.has(result.chunk.id) && result.similarity > 0.9) {
|
|
857
|
+
seenChunkIds.add(result.chunk.id);
|
|
858
|
+
allResults.push(result);
|
|
859
|
+
preferenceHits++;
|
|
860
|
+
}
|
|
861
|
+
}
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
console.log('[RAG Filter] Preference boost added', preferenceHits, 'unique chunks');
|
|
865
|
+
}
|
|
866
|
+
|
|
867
|
+
// Expand results based on similarity score (high score = more context)
|
|
868
|
+
const expandedChunks = this.expandResults(allResults);
|
|
869
|
+
|
|
870
|
+
// Build filtered observations with preserved structure (thread + date grouping)
|
|
871
|
+
const filteredObservations = rebuildObservationsWithStructure(expandedChunks);
|
|
872
|
+
|
|
873
|
+
// Build the new context with filtered observations
|
|
874
|
+
let newContext = `
|
|
875
|
+
The following observations block contains your memory of past conversations with this user. Use these observations to provide personalized, contextually relevant responses.
|
|
876
|
+
|
|
877
|
+
<observations>
|
|
878
|
+
${filteredObservations}
|
|
879
|
+
</observations>`;
|
|
880
|
+
|
|
881
|
+
// Optionally include patterns (already compressed, usually worth including)
|
|
882
|
+
if (this.includePatterns) {
|
|
883
|
+
const patterns = extractPatternsBlock(fullContext);
|
|
884
|
+
if (patterns) {
|
|
885
|
+
newContext += `\n\n<patterns>\n${patterns}\n</patterns>`;
|
|
886
|
+
}
|
|
887
|
+
}
|
|
888
|
+
|
|
889
|
+
// Optionally include current-task
|
|
890
|
+
if (this.includeCurrentTask) {
|
|
891
|
+
const currentTask = extractCurrentTaskBlock(fullContext);
|
|
892
|
+
if (currentTask) {
|
|
893
|
+
newContext += `\n\n<current-task>\n${currentTask}\n</current-task>`;
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
|
|
897
|
+
// Optionally include suggested-response
|
|
898
|
+
if (this.includeSuggestedResponse) {
|
|
899
|
+
const suggestedResponse = extractSuggestedResponseBlock(fullContext);
|
|
900
|
+
if (suggestedResponse) {
|
|
901
|
+
newContext += `\n\n<suggested-response>\n${suggestedResponse}\n</suggested-response>`;
|
|
902
|
+
}
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
// Replace OM's system message with our filtered version
|
|
906
|
+
// Get all system messages, replace OM's, and set them all back
|
|
907
|
+
const allSystemMessages = messageList.getAllSystemMessages();
|
|
908
|
+
const updatedSystemMessages = allSystemMessages.map(msg => {
|
|
909
|
+
// Replace the OM message with our filtered version
|
|
910
|
+
if (msg === omSystemMessage) {
|
|
911
|
+
return {
|
|
912
|
+
role: 'system' as const,
|
|
913
|
+
content: newContext,
|
|
914
|
+
};
|
|
915
|
+
}
|
|
916
|
+
return msg;
|
|
917
|
+
});
|
|
918
|
+
messageList.replaceAllSystemMessages(updatedSystemMessages);
|
|
919
|
+
console.log('[RAG Filter] Replaced system messages. New context length:', newContext.length, 'chars');
|
|
920
|
+
|
|
921
|
+
// Store stats in state for debugging
|
|
922
|
+
state.ragStats = {
|
|
923
|
+
originalLines: this.allParsedObservations.length,
|
|
924
|
+
matchedLines: mainQueryResultCount,
|
|
925
|
+
expandedQueryCount: queriesToRun.length,
|
|
926
|
+
totalMatches: allResults.length,
|
|
927
|
+
expandedLines: expandedChunks.length,
|
|
928
|
+
topSimilarity,
|
|
929
|
+
query: userQuery.slice(0, 100),
|
|
930
|
+
};
|
|
931
|
+
|
|
932
|
+
return messageList;
|
|
933
|
+
}
|
|
934
|
+
|
|
935
|
+
/**
|
|
936
|
+
* Index observations into the vector store with content-based caching
|
|
937
|
+
*/
|
|
938
|
+
private async indexObservations(observations: string, state: Record<string, unknown>): Promise<void> {
|
|
939
|
+
const parsedObs = parseObservationLines(observations);
|
|
940
|
+
if (parsedObs.length === 0) return;
|
|
941
|
+
|
|
942
|
+
// Store all parsed observations for neighbor lookup during expansion
|
|
943
|
+
this.allParsedObservations = parsedObs;
|
|
944
|
+
|
|
945
|
+
// Get resourceId from state if available
|
|
946
|
+
const resourceId = state.resourceId as string | undefined;
|
|
947
|
+
|
|
948
|
+
// Separate cached vs uncached observations
|
|
949
|
+
const uncachedObs: ParsedObservation[] = [];
|
|
950
|
+
const cachedChunks: ObservationChunk[] = [];
|
|
951
|
+
const obsHashes: string[] = [];
|
|
952
|
+
|
|
953
|
+
for (const obs of parsedObs) {
|
|
954
|
+
const hash = await this.hashContent(obs.line);
|
|
955
|
+
obsHashes.push(hash);
|
|
956
|
+
|
|
957
|
+
const cachedEmbedding = this.getCachedEmbedding(hash);
|
|
958
|
+
if (cachedEmbedding) {
|
|
959
|
+
// Use cached embedding
|
|
960
|
+
cachedChunks.push({
|
|
961
|
+
id: `obs_${hash}`,
|
|
962
|
+
embedding: cachedEmbedding,
|
|
963
|
+
metadata: {
|
|
964
|
+
threadId: obs.threadId,
|
|
965
|
+
resourceId,
|
|
966
|
+
dateGroup: obs.dateGroup,
|
|
967
|
+
time: obs.time,
|
|
968
|
+
labels: obs.labels,
|
|
969
|
+
lineIndex: obs.lineIndex,
|
|
970
|
+
content: obs.line,
|
|
971
|
+
},
|
|
972
|
+
});
|
|
973
|
+
} else {
|
|
974
|
+
uncachedObs.push(obs);
|
|
975
|
+
}
|
|
976
|
+
}
|
|
977
|
+
|
|
978
|
+
console.log(`[RAG Filter] Cache: ${cachedChunks.length} hits, ${uncachedObs.length} misses`);
|
|
979
|
+
|
|
980
|
+
// Embed uncached observations in batches
|
|
981
|
+
if (uncachedObs.length > 0) {
|
|
982
|
+
const batchSize = 25;
|
|
983
|
+
for (let b = 0; b < uncachedObs.length; b += batchSize) {
|
|
984
|
+
const batch = uncachedObs.slice(b, b + batchSize);
|
|
985
|
+
|
|
986
|
+
const { embeddings } = await embedMany({
|
|
987
|
+
model: this.embedder,
|
|
988
|
+
values: batch.map(l => l.line),
|
|
989
|
+
});
|
|
990
|
+
|
|
991
|
+
// Create chunks and save to cache
|
|
992
|
+
for (let i = 0; i < batch.length; i++) {
|
|
993
|
+
const obs = batch[i];
|
|
994
|
+
const hash = await this.hashContent(obs.line);
|
|
995
|
+
const embedding = embeddings[i];
|
|
996
|
+
|
|
997
|
+
// Save to cache
|
|
998
|
+
this.saveEmbeddingToCache(hash, embedding);
|
|
999
|
+
|
|
1000
|
+
cachedChunks.push({
|
|
1001
|
+
id: `obs_${hash}`,
|
|
1002
|
+
embedding,
|
|
1003
|
+
metadata: {
|
|
1004
|
+
threadId: obs.threadId,
|
|
1005
|
+
resourceId,
|
|
1006
|
+
dateGroup: obs.dateGroup,
|
|
1007
|
+
time: obs.time,
|
|
1008
|
+
labels: obs.labels,
|
|
1009
|
+
lineIndex: obs.lineIndex,
|
|
1010
|
+
content: obs.line,
|
|
1011
|
+
},
|
|
1012
|
+
});
|
|
1013
|
+
}
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
this.vectorStore.upsert(cachedChunks);
|
|
1018
|
+
}
|
|
1019
|
+
|
|
1020
|
+
/**
|
|
1021
|
+
* Expand search results by including neighboring observations based on similarity score.
|
|
1022
|
+
* High score (≥0.6): Include ±2 lines of context
|
|
1023
|
+
* Medium score (0.4-0.6): Include ±1 line of context
|
|
1024
|
+
* Low score (<0.4): Just the matched line
|
|
1025
|
+
*/
|
|
1026
|
+
private expandResults(
|
|
1027
|
+
results: Array<{ chunk: ObservationChunk; similarity: number }>,
|
|
1028
|
+
): Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }> {
|
|
1029
|
+
if (!this.expandContext || this.allParsedObservations.length === 0) {
|
|
1030
|
+
// Return results as-is without expansion
|
|
1031
|
+
return results.map(r => r.chunk);
|
|
1032
|
+
}
|
|
1033
|
+
|
|
1034
|
+
// Create a lookup for all observations by threadId and lineIndex
|
|
1035
|
+
const obsLookup = new Map<string, ParsedObservation[]>();
|
|
1036
|
+
for (const obs of this.allParsedObservations) {
|
|
1037
|
+
const key = obs.threadId;
|
|
1038
|
+
if (!obsLookup.has(key)) {
|
|
1039
|
+
obsLookup.set(key, []);
|
|
1040
|
+
}
|
|
1041
|
+
obsLookup.get(key)!.push(obs);
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
// Sort each thread's observations by lineIndex
|
|
1045
|
+
for (const [, obsList] of Array.from(obsLookup.entries())) {
|
|
1046
|
+
obsList.sort((a, b) => a.lineIndex - b.lineIndex);
|
|
1047
|
+
}
|
|
1048
|
+
|
|
1049
|
+
// Track which lines we've already included (by threadId + lineIndex)
|
|
1050
|
+
const includedLines = new Set<string>();
|
|
1051
|
+
const expandedChunks: Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }> = [];
|
|
1052
|
+
|
|
1053
|
+
for (const result of results) {
|
|
1054
|
+
const { chunk, similarity } = result;
|
|
1055
|
+
const { threadId, lineIndex, resourceId } = chunk.metadata;
|
|
1056
|
+
|
|
1057
|
+
// Determine expansion range based on score
|
|
1058
|
+
// High score = include FULL thread, medium = ±1 line, low = just the match
|
|
1059
|
+
const includeFullThread = similarity >= this.highScoreThreshold;
|
|
1060
|
+
let range = 0;
|
|
1061
|
+
if (!includeFullThread && similarity >= this.mediumScoreThreshold) {
|
|
1062
|
+
range = 2; // ±1 line for medium confidence
|
|
1063
|
+
}
|
|
1064
|
+
// range = 0 for low confidence (just the match)
|
|
1065
|
+
|
|
1066
|
+
// Get observations for this thread
|
|
1067
|
+
const threadObs = obsLookup.get(threadId) || [];
|
|
1068
|
+
if (threadObs.length === 0) {
|
|
1069
|
+
// Fallback: just include the original chunk
|
|
1070
|
+
const lineKey = `${threadId}:${lineIndex}`;
|
|
1071
|
+
if (!includedLines.has(lineKey)) {
|
|
1072
|
+
includedLines.add(lineKey);
|
|
1073
|
+
expandedChunks.push(chunk);
|
|
1074
|
+
}
|
|
1075
|
+
continue;
|
|
1076
|
+
}
|
|
1077
|
+
|
|
1078
|
+
// Find observations within range (or all if includeFullThread)
|
|
1079
|
+
for (const obs of threadObs) {
|
|
1080
|
+
const inRange = includeFullThread || (obs.lineIndex >= lineIndex - range && obs.lineIndex <= lineIndex + range);
|
|
1081
|
+
if (inRange) {
|
|
1082
|
+
const lineKey = `${threadId}:${obs.lineIndex}`;
|
|
1083
|
+
if (!includedLines.has(lineKey)) {
|
|
1084
|
+
includedLines.add(lineKey);
|
|
1085
|
+
expandedChunks.push({
|
|
1086
|
+
metadata: {
|
|
1087
|
+
threadId: obs.threadId,
|
|
1088
|
+
resourceId,
|
|
1089
|
+
dateGroup: obs.dateGroup,
|
|
1090
|
+
time: obs.time,
|
|
1091
|
+
labels: obs.labels,
|
|
1092
|
+
lineIndex: obs.lineIndex,
|
|
1093
|
+
content: obs.line,
|
|
1094
|
+
},
|
|
1095
|
+
});
|
|
1096
|
+
}
|
|
1097
|
+
}
|
|
1098
|
+
}
|
|
1099
|
+
}
|
|
1100
|
+
|
|
1101
|
+
// Sort expanded chunks by threadId, dateGroup, then lineIndex for proper reconstruction
|
|
1102
|
+
expandedChunks.sort((a, b) => {
|
|
1103
|
+
if (a.metadata.threadId !== b.metadata.threadId) {
|
|
1104
|
+
return a.metadata.threadId.localeCompare(b.metadata.threadId);
|
|
1105
|
+
}
|
|
1106
|
+
if (a.metadata.dateGroup !== b.metadata.dateGroup) {
|
|
1107
|
+
return a.metadata.dateGroup.localeCompare(b.metadata.dateGroup);
|
|
1108
|
+
}
|
|
1109
|
+
return a.metadata.lineIndex - b.metadata.lineIndex;
|
|
1110
|
+
});
|
|
1111
|
+
|
|
1112
|
+
console.log(
|
|
1113
|
+
`[RAG Filter] Context expansion: ${results.length} matches → ${expandedChunks.length} lines (range based on score)`,
|
|
1114
|
+
);
|
|
1115
|
+
|
|
1116
|
+
return expandedChunks;
|
|
1117
|
+
}
|
|
1118
|
+
|
|
1119
|
+
/**
|
|
1120
|
+
* Process output - index new observations after OM creates them
|
|
1121
|
+
*
|
|
1122
|
+
* Note: This is called after OM's processOutputResult, so new observations
|
|
1123
|
+
* should be available in the OM record. For now, we'll skip this and
|
|
1124
|
+
* rely on indexing during processInputStep.
|
|
1125
|
+
*/
|
|
1126
|
+
async processOutputResult(args: ProcessOutputResultArgs) {
|
|
1127
|
+
// For the initial implementation, we index observations lazily during
|
|
1128
|
+
// processInputStep. This avoids the complexity of coordinating with OM's
|
|
1129
|
+
// output processing.
|
|
1130
|
+
//
|
|
1131
|
+
// Future enhancement: Index new observations here for better performance
|
|
1132
|
+
// on subsequent queries within the same session.
|
|
1133
|
+
return args.messageList;
|
|
1134
|
+
}
|
|
1135
|
+
}
|
|
1136
|
+
|
|
1137
|
+
/**
|
|
1138
|
+
* Create an ObservationSemanticFilter processor
|
|
1139
|
+
*/
|
|
1140
|
+
export function createObservationSemanticFilter(config: ObservationSemanticFilterConfig): ObservationSemanticFilter {
|
|
1141
|
+
return new ObservationSemanticFilter(config);
|
|
1142
|
+
}
|