@mastra/longmemeval 0.0.0-error-handler-fix-20251020202607 → 0.0.0-execa-dynamic-import-20260304221256

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. package/CHANGELOG.md +847 -6
  2. package/LICENSE.md +15 -0
  3. package/docs/INVESTIGATION_WORKFLOW.md +314 -0
  4. package/package.json +34 -26
  5. package/scripts/download.ts +22 -1
  6. package/scripts/generate-wm-templates.ts +2 -1
  7. package/src/cli.ts +967 -155
  8. package/src/commands/__tests__/prepare.test.ts +11 -6
  9. package/src/commands/__tests__/run.test.ts +10 -8
  10. package/src/commands/clean.ts +147 -0
  11. package/src/commands/deterministic-ids.ts +126 -0
  12. package/src/commands/find-prohibited.ts +299 -0
  13. package/src/commands/investigate.ts +2437 -0
  14. package/src/commands/list-partial.ts +137 -0
  15. package/src/commands/obscure-thread-ids.ts +237 -0
  16. package/src/commands/partial-results.ts +198 -0
  17. package/src/commands/precompute-embeddings.ts +302 -0
  18. package/src/commands/prepare.ts +732 -105
  19. package/src/commands/reconcile.ts +383 -0
  20. package/src/commands/run.ts +1240 -121
  21. package/src/commands/sessions.ts +126 -0
  22. package/src/commands/sync.ts +152 -0
  23. package/src/commands/test-cleaned.ts +53 -0
  24. package/src/commands/test-incremental.ts +120 -0
  25. package/src/commands/test-messages.ts +95 -0
  26. package/src/commands/test-prohibited.ts +244 -0
  27. package/src/commands/test-thread.ts +136 -0
  28. package/src/commands/tokens.ts +625 -0
  29. package/src/config.ts +1253 -72
  30. package/src/data/types.ts +48 -6
  31. package/src/embeddings/cached-openai-embedding-model.ts +0 -1
  32. package/src/evaluation/longmemeval-metric.ts +186 -103
  33. package/src/processors/content-sanitizer.ts +153 -0
  34. package/src/processors/date-injector.ts +111 -0
  35. package/src/processors/index.ts +3 -0
  36. package/src/processors/observation-semantic-filter.ts +1142 -0
  37. package/src/retry-model.ts +5 -3
  38. package/src/storage/__tests__/benchmark-store.test.ts +18 -35
  39. package/src/storage/benchmark-store.ts +140 -153
  40. package/src/storage/benchmark-vector.ts +1 -1
  41. package/src/storage/index.ts +1 -0
  42. package/src/storage/persistable-inmemory.ts +133 -0
  43. package/src/test-utils/mock-model.ts +35 -0
@@ -0,0 +1,1142 @@
1
+ /**
2
+ * ObservationSemanticFilter - RAG-based filtering for OM observations
3
+ *
4
+ * This processor works alongside ObservationalMemory to filter observations
5
+ * based on semantic similarity to the user's query. It's designed to help
6
+ * models like GPT-4o that struggle with dense context by providing only
7
+ * the most relevant observations.
8
+ *
9
+ * Flow:
10
+ * 1. OM's processInputStep loads observations and stores them in state
11
+ * 2. This processor's processInputStep:
12
+ * - Embeds the user's query
13
+ * - Retrieves relevant observation chunks from vector store
14
+ * - Injects filtered observations into context (replacing OM's full injection)
15
+ *
16
+ * 3. OM's processOutputResult creates new observations
17
+ * 4. This processor's processOutputResult:
18
+ * - Chunks new observations by line
19
+ * - Embeds and stores chunks in vector store
20
+ */
21
+
22
+ import { embed, embedMany } from 'ai';
23
+ import type { EmbeddingModel } from 'ai';
24
+ import type { Processor, ProcessInputStepArgs, ProcessOutputResultArgs } from '@mastra/core/processors';
25
+ import type { MastraDBMessage } from '@mastra/core/agent';
26
+ import { existsSync, readFileSync, writeFileSync, mkdirSync } from 'fs';
27
+ import { join } from 'path';
28
+ import xxhash from 'xxhash-wasm';
29
+
30
+ // ============================================================================
31
+ // Types
32
+ // ============================================================================
33
+
34
+ /**
35
+ * Metadata stored with each observation chunk in the vector store
36
+ */
37
+ export interface ObservationChunkMetadata {
38
+ /** The thread this observation came from */
39
+ threadId: string;
40
+ /** Resource ID for cross-thread queries */
41
+ resourceId?: string;
42
+ /** The date group header this observation belongs to (e.g., "Date: May 30, 2023") */
43
+ dateGroup: string;
44
+ /** Time extracted from the observation line (e.g., "17:27") */
45
+ time?: string;
46
+ /** Labels extracted from the observation (e.g., [preference], [fact]) */
47
+ labels?: string[];
48
+ /** Index of this line within the observation block */
49
+ lineIndex: number;
50
+ /** The full observation line content */
51
+ content: string;
52
+ }
53
+
54
+ /**
55
+ * A chunk stored in the vector store
56
+ */
57
+ export interface ObservationChunk {
58
+ id: string;
59
+ embedding: number[];
60
+ metadata: ObservationChunkMetadata;
61
+ }
62
+
63
+ /**
64
+ * Configuration for the ObservationSemanticFilter
65
+ */
66
+ export interface ObservationSemanticFilterConfig {
67
+ /**
68
+ * Embedding model to use for chunking and retrieval
69
+ * @example fastembed.small from @mastra/fastembed
70
+ */
71
+ embedder: EmbeddingModel<string>;
72
+
73
+ /**
74
+ * Number of top chunks to retrieve per query
75
+ * @default 50
76
+ */
77
+ topK?: number;
78
+
79
+ /**
80
+ * Minimum similarity score (0-1) for a chunk to be included
81
+ * @default 0.3
82
+ */
83
+ minSimilarity?: number;
84
+
85
+ /**
86
+ * Whether to include <current-task> in filtered output
87
+ * @default false (saves tokens for benchmarks)
88
+ */
89
+ includeCurrentTask?: boolean;
90
+
91
+ /**
92
+ * Whether to include <suggested-response> in filtered output
93
+ * @default false (saves tokens for benchmarks)
94
+ */
95
+ includeSuggestedResponse?: boolean;
96
+
97
+ /**
98
+ * Whether to include <patterns> in filtered output
99
+ * @default true (patterns are already compressed)
100
+ */
101
+ includePatterns?: boolean;
102
+
103
+ /**
104
+ * Directory to cache embeddings on disk.
105
+ * Embeddings are stored by content hash (xxhash) so they can be reused across configs.
106
+ * @default undefined (no caching)
107
+ */
108
+ cacheDir?: string;
109
+
110
+ /**
111
+ * Enable score-based context expansion.
112
+ * High-scoring matches will include surrounding observation lines.
113
+ * @default true
114
+ */
115
+ expandContext?: boolean;
116
+
117
+ /**
118
+ * Similarity threshold for high-confidence matches (±2 lines)
119
+ * @default 0.6
120
+ */
121
+ highScoreThreshold?: number;
122
+
123
+ /**
124
+ * Similarity threshold for medium-confidence matches (±1 line)
125
+ * @default 0.4
126
+ */
127
+ mediumScoreThreshold?: number;
128
+
129
+ /**
130
+ * Enable preference boost queries.
131
+ * Runs additional searches for "user prefers", "user preference", etc.
132
+ * to ensure user preferences are always included in context.
133
+ * @default true
134
+ */
135
+ preferenceBoost?: boolean;
136
+
137
+ /**
138
+ * TopK for preference boost queries
139
+ * @default 50
140
+ */
141
+ preferenceBoostTopK?: number;
142
+
143
+ /**
144
+ * Minimum similarity for preference boost queries (lower than main query)
145
+ * @default 0.35
146
+ */
147
+ preferenceBoostMinSimilarity?: number;
148
+
149
+ /**
150
+ * Enable query expansion for complex/multi-part questions.
151
+ * Extracts quoted text, sentences, and key phrases as separate queries.
152
+ * @default true
153
+ */
154
+ expandQueries?: boolean;
155
+
156
+ /**
157
+ * TopK for each expanded query (smaller than main query since results are merged)
158
+ * @default 20
159
+ */
160
+ queryExpansionTopK?: number;
161
+ }
162
+
163
+ /**
164
+ * In-memory vector store for observation chunks
165
+ * Simple implementation for experimentation - can be replaced with LibSQL later
166
+ */
167
+ export class InMemoryVectorStore {
168
+ private chunks: Map<string, ObservationChunk> = new Map();
169
+
170
+ /**
171
+ * Add or update chunks in the store
172
+ */
173
+ upsert(chunks: ObservationChunk[]): void {
174
+ for (const chunk of chunks) {
175
+ this.chunks.set(chunk.id, chunk);
176
+ }
177
+ }
178
+
179
+ /**
180
+ * Query chunks by similarity to a query embedding
181
+ */
182
+ query(
183
+ queryEmbedding: number[],
184
+ options: { topK: number; minSimilarity: number; resourceId?: string },
185
+ ): Array<{ chunk: ObservationChunk; similarity: number }> {
186
+ const results: Array<{ chunk: ObservationChunk; similarity: number }> = [];
187
+
188
+ for (const chunk of Array.from(this.chunks.values())) {
189
+ // Filter by resourceId if provided
190
+ if (options.resourceId && chunk.metadata.resourceId !== options.resourceId) {
191
+ continue;
192
+ }
193
+
194
+ const similarity = this.cosineSimilarity(queryEmbedding, chunk.embedding);
195
+ if (similarity >= options.minSimilarity) {
196
+ results.push({ chunk, similarity });
197
+ }
198
+ }
199
+
200
+ // Sort by similarity descending and take topK
201
+ return results.sort((a, b) => b.similarity - a.similarity).slice(0, options.topK);
202
+ }
203
+
204
+ /**
205
+ * Get all chunks (for debugging/inspection)
206
+ */
207
+ getAll(): ObservationChunk[] {
208
+ return Array.from(this.chunks.values());
209
+ }
210
+
211
+ /**
212
+ * Clear all chunks
213
+ */
214
+ clear(): void {
215
+ this.chunks.clear();
216
+ }
217
+
218
+ /**
219
+ * Get chunk count
220
+ */
221
+ get size(): number {
222
+ return this.chunks.size;
223
+ }
224
+
225
+ /**
226
+ * Cosine similarity between two vectors
227
+ */
228
+ private cosineSimilarity(a: number[], b: number[]): number {
229
+ if (a.length !== b.length) return 0;
230
+
231
+ let dotProduct = 0;
232
+ let normA = 0;
233
+ let normB = 0;
234
+
235
+ for (let i = 0; i < a.length; i++) {
236
+ dotProduct += a[i] * b[i];
237
+ normA += a[i] * a[i];
238
+ normB += b[i] * b[i];
239
+ }
240
+
241
+ const magnitude = Math.sqrt(normA) * Math.sqrt(normB);
242
+ return magnitude === 0 ? 0 : dotProduct / magnitude;
243
+ }
244
+
245
+ /**
246
+ * Serialize store to JSON (for persistence)
247
+ */
248
+ toJSON(): { chunks: ObservationChunk[] } {
249
+ return { chunks: Array.from(this.chunks.values()) };
250
+ }
251
+
252
+ /**
253
+ * Load store from JSON
254
+ */
255
+ static fromJSON(data: { chunks: ObservationChunk[] }): InMemoryVectorStore {
256
+ const store = new InMemoryVectorStore();
257
+ store.upsert(data.chunks);
258
+ return store;
259
+ }
260
+ }
261
+
262
+ // ============================================================================
263
+ // Parsing Helpers
264
+ // ============================================================================
265
+
266
+ /**
267
+ * Parsed observation with full context (thread, date group, content)
268
+ */
269
+ export interface ParsedObservation {
270
+ line: string;
271
+ lineIndex: number;
272
+ threadId: string;
273
+ dateGroup: string; // The date header (e.g., "Date: May 30, 2023")
274
+ time?: string; // Time extracted from line (e.g., "17:27")
275
+ labels?: string[];
276
+ }
277
+
278
+ /**
279
+ * Parse observation lines from the <observations> block, preserving thread and date context
280
+ * Each line is one observation item (emoji + time + content + labels)
281
+ */
282
+ export function parseObservationLines(observations: string): ParsedObservation[] {
283
+ const lines = observations.split('\n');
284
+ const results: ParsedObservation[] = [];
285
+
286
+ let currentThreadId = 'default';
287
+ let currentDateGroup = 'Unknown Date';
288
+
289
+ for (let i = 0; i < lines.length; i++) {
290
+ const line = lines[i].trim();
291
+ if (!line) continue;
292
+
293
+ // Check for thread headers
294
+ const threadMatch = line.match(/^<thread\s+id="([^"]+)"/i) || line.match(/^<other-conversation\s+id="([^"]+)"/i);
295
+ if (threadMatch) {
296
+ currentThreadId = threadMatch[1];
297
+ continue;
298
+ }
299
+
300
+ // Check for thread closing tags
301
+ if (line.match(/^<\/thread>/i) || line.match(/^<\/other-conversation>/i)) {
302
+ currentThreadId = 'default';
303
+ continue;
304
+ }
305
+
306
+ // Check for date headers
307
+ // Format 1: "## 2023-05-15" or "# 2023-05-15"
308
+ const dateHeaderMatch1 = line.match(/^##?\s*(\d{4}-\d{2}-\d{2})/);
309
+ if (dateHeaderMatch1) {
310
+ currentDateGroup = `Date: ${dateHeaderMatch1[1]}`;
311
+ continue;
312
+ }
313
+ // Format 2: "Date: May 30, 2023" or "Date: Dec 20, 2022"
314
+ const dateHeaderMatch2 = line.match(/^(Date:\s+.+)$/i);
315
+ if (dateHeaderMatch2) {
316
+ currentDateGroup = dateHeaderMatch2[1];
317
+ continue;
318
+ }
319
+
320
+ // Skip other XML tags
321
+ if (line.startsWith('<') || line.startsWith('</')) {
322
+ continue;
323
+ }
324
+
325
+ // This is an observation line - extract time and labels
326
+ // Time format: (HH:MM) at the start after emoji
327
+ const timeMatch = line.match(/\((\d{2}:\d{2})\)/);
328
+ const time = timeMatch ? timeMatch[1] : undefined;
329
+
330
+ // Extract labels (format: [label])
331
+ const labelMatches = line.match(/\[([^\]]+)\]/g);
332
+ const labels = labelMatches?.map(l => l.slice(1, -1));
333
+
334
+ results.push({
335
+ line,
336
+ lineIndex: i,
337
+ threadId: currentThreadId,
338
+ dateGroup: currentDateGroup,
339
+ time,
340
+ labels,
341
+ });
342
+ }
343
+
344
+ return results;
345
+ }
346
+
347
+ /**
348
+ * Rebuild observations with thread and date structure from retrieved chunks
349
+ */
350
+ export function rebuildObservationsWithStructure(
351
+ chunks: Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }>,
352
+ ): string {
353
+ // Group by thread, then by date
354
+ const threadGroups = new Map<string, Map<string, string[]>>();
355
+
356
+ for (const chunk of chunks) {
357
+ const { threadId, dateGroup, content } = chunk.metadata;
358
+
359
+ if (!threadGroups.has(threadId)) {
360
+ threadGroups.set(threadId, new Map());
361
+ }
362
+ const dateGroups = threadGroups.get(threadId)!;
363
+
364
+ if (!dateGroups.has(dateGroup)) {
365
+ dateGroups.set(dateGroup, []);
366
+ }
367
+ dateGroups.get(dateGroup)!.push(content);
368
+ }
369
+
370
+ // Build output with structure
371
+ const parts: string[] = [];
372
+
373
+ for (const [threadId, dateGroups] of Array.from(threadGroups.entries())) {
374
+ // Add thread wrapper if not default
375
+ if (threadId !== 'default') {
376
+ parts.push(`<thread id="${threadId}">`);
377
+ }
378
+
379
+ for (const [dateGroup, observations] of Array.from(dateGroups.entries())) {
380
+ parts.push(dateGroup);
381
+ parts.push(...observations);
382
+ }
383
+
384
+ if (threadId !== 'default') {
385
+ parts.push('</thread>');
386
+ }
387
+ }
388
+
389
+ return parts.join('\n');
390
+ }
391
+
392
+ /**
393
+ * Extract the <observations> content from formatted context
394
+ */
395
+ export function extractObservationsBlock(formattedContext: string): string | null {
396
+ const match = formattedContext.match(/<observations>([\s\S]*?)<\/observations>/);
397
+ return match ? match[1].trim() : null;
398
+ }
399
+
400
+ /**
401
+ * Extract the <patterns> content from formatted context
402
+ */
403
+ export function extractPatternsBlock(formattedContext: string): string | null {
404
+ const match = formattedContext.match(/<patterns>([\s\S]*?)<\/patterns>/);
405
+ return match ? match[1].trim() : null;
406
+ }
407
+
408
+ /**
409
+ * Extract the <current-task> content from formatted context
410
+ */
411
+ export function extractCurrentTaskBlock(formattedContext: string): string | null {
412
+ const match = formattedContext.match(/<current-task>([\s\S]*?)<\/current-task>/);
413
+ return match ? match[1].trim() : null;
414
+ }
415
+
416
+ /**
417
+ * Extract the <suggested-response> content from formatted context
418
+ */
419
+ export function extractSuggestedResponseBlock(formattedContext: string): string | null {
420
+ const match = formattedContext.match(/<suggested-response>([\s\S]*?)<\/suggested-response>/);
421
+ return match ? match[1].trim() : null;
422
+ }
423
+
424
+ /**
425
+ * Get the most recent user message from the message list
426
+ */
427
+ export function getLatestUserMessage(messages: MastraDBMessage[]): string | null {
428
+ for (let i = messages.length - 1; i >= 0; i--) {
429
+ const msg = messages[i];
430
+ if (msg.role === 'user') {
431
+ const content = msg.content;
432
+
433
+ // Handle string content (legacy format)
434
+ if (typeof content === 'string') {
435
+ return content;
436
+ }
437
+
438
+ // Handle MastraMessageContentV2 format (format: 2 with parts array)
439
+ if (content && typeof content === 'object' && 'format' in content && content.format === 2) {
440
+ // First check if there's a direct content string
441
+ if (typeof content.content === 'string' && content.content) {
442
+ return content.content;
443
+ }
444
+ // Otherwise extract from parts
445
+ if (Array.isArray(content.parts)) {
446
+ const textParts = content.parts
447
+ .filter(
448
+ (p): p is { type: 'text'; text: string } =>
449
+ p && typeof p === 'object' && p.type === 'text' && typeof p.text === 'string',
450
+ )
451
+ .map(p => p.text);
452
+ if (textParts.length > 0) {
453
+ return textParts.join(' ');
454
+ }
455
+ }
456
+ }
457
+
458
+ // Handle plain array content (older format)
459
+ if (Array.isArray(content)) {
460
+ const textParts = content
461
+ .filter(
462
+ (p): p is { type: 'text'; text: string } =>
463
+ p && typeof p === 'object' && p.type === 'text' && typeof p.text === 'string',
464
+ )
465
+ .map(p => p.text);
466
+ return textParts.join(' ') || null;
467
+ }
468
+ }
469
+ }
470
+ return null;
471
+ }
472
+
473
+ // ============================================================================
474
+ // Processor Implementation
475
+ // ============================================================================
476
+
477
+ /**
478
+ * ObservationSemanticFilter processor
479
+ *
480
+ * Works with ObservationalMemory to filter observations based on semantic
481
+ * similarity to the user's query.
482
+ */
483
+ export class ObservationSemanticFilter implements Processor<'observation-semantic-filter'> {
484
+ readonly id = 'observation-semantic-filter' as const;
485
+ readonly name = 'Observation Semantic Filter';
486
+
487
+ private embedder: EmbeddingModel<string>;
488
+ private vectorStore: InMemoryVectorStore;
489
+ private topK: number;
490
+ private minSimilarity: number;
491
+ private includeCurrentTask: boolean;
492
+ private includeSuggestedResponse: boolean;
493
+ private includePatterns: boolean;
494
+ private cacheDir?: string;
495
+ private hasher = xxhash();
496
+ private embeddingCache = new Map<string, number[]>(); // hash -> embedding
497
+
498
+ // Context expansion settings
499
+ private expandContext: boolean;
500
+ private highScoreThreshold: number;
501
+ private mediumScoreThreshold: number;
502
+
503
+ // Preference boost settings
504
+ private preferenceBoost: boolean;
505
+ private preferenceBoostTopK: number;
506
+ private preferenceBoostMinSimilarity: number;
507
+
508
+ // Store all parsed observations for neighbor lookup during expansion
509
+ private allParsedObservations: ParsedObservation[] = [];
510
+
511
+ // Query expansion settings
512
+ private expandQueries: boolean;
513
+ private queryExpansionTopK: number;
514
+
515
+ constructor(config: ObservationSemanticFilterConfig) {
516
+ this.embedder = config.embedder;
517
+ this.vectorStore = new InMemoryVectorStore();
518
+ this.topK = config.topK ?? 30;
519
+ this.minSimilarity = config.minSimilarity ?? 0.4;
520
+ this.includeCurrentTask = config.includeCurrentTask ?? false;
521
+ this.includeSuggestedResponse = config.includeSuggestedResponse ?? false;
522
+ this.includePatterns = config.includePatterns ?? false;
523
+ this.cacheDir = config.cacheDir;
524
+ this.expandContext = config.expandContext ?? true;
525
+ this.highScoreThreshold = config.highScoreThreshold ?? 0.8;
526
+ this.mediumScoreThreshold = config.mediumScoreThreshold ?? 0.4;
527
+ this.preferenceBoost = config.preferenceBoost ?? true;
528
+ this.preferenceBoostTopK = config.preferenceBoostTopK ?? 10;
529
+ this.preferenceBoostMinSimilarity = config.preferenceBoostMinSimilarity ?? 0.8;
530
+ this.expandQueries = config.expandQueries ?? false;
531
+ this.queryExpansionTopK = config.queryExpansionTopK ?? 5;
532
+
533
+ // Ensure cache directory exists
534
+ if (this.cacheDir && !existsSync(this.cacheDir)) {
535
+ mkdirSync(this.cacheDir, { recursive: true });
536
+ }
537
+ }
538
+
539
+ /**
540
+ * Hash content to create a cache key using xxhash
541
+ */
542
+ private async hashContent(content: string): Promise<string> {
543
+ const hasher = await this.hasher;
544
+ return hasher.h32ToString(content);
545
+ }
546
+
547
+ /**
548
+ * Get cached embedding for content, or null if not cached
549
+ */
550
+ private getCachedEmbedding(hash: string): number[] | null {
551
+ // Check in-memory cache first
552
+ if (this.embeddingCache.has(hash)) {
553
+ return this.embeddingCache.get(hash)!;
554
+ }
555
+
556
+ // Check disk cache
557
+ if (this.cacheDir) {
558
+ const cachePath = join(this.cacheDir, `${hash}.json`);
559
+ if (existsSync(cachePath)) {
560
+ try {
561
+ const embedding = JSON.parse(readFileSync(cachePath, 'utf-8'));
562
+ this.embeddingCache.set(hash, embedding); // Store in memory for future lookups
563
+ return embedding;
564
+ } catch {
565
+ // Ignore corrupted cache files
566
+ }
567
+ }
568
+ }
569
+
570
+ return null;
571
+ }
572
+
573
+ /**
574
+ * Save embedding to cache (both in-memory and disk)
575
+ */
576
+ private saveEmbeddingToCache(hash: string, embedding: number[]): void {
577
+ this.embeddingCache.set(hash, embedding);
578
+
579
+ if (this.cacheDir) {
580
+ const cachePath = join(this.cacheDir, `${hash}.json`);
581
+ try {
582
+ writeFileSync(cachePath, JSON.stringify(embedding));
583
+ } catch {
584
+ // Ignore write errors
585
+ }
586
+ }
587
+ }
588
+
589
+ /**
590
+ * Get the vector store (for external access/persistence)
591
+ */
592
+ getVectorStore(): InMemoryVectorStore {
593
+ return this.vectorStore;
594
+ }
595
+
596
+ /**
597
+ * Set the vector store (for loading persisted data)
598
+ */
599
+ setVectorStore(store: InMemoryVectorStore): void {
600
+ this.vectorStore = store;
601
+ }
602
+
603
+ /**
604
+ * Expand a complex query into multiple sub-queries for better retrieval.
605
+ * Extracts:
606
+ * - Quoted text (e.g., "The Mountain Meditation")
607
+ * - Individual sentences
608
+ * - Key noun phrases
609
+ *
610
+ * @returns Array of query strings to embed and search
611
+ */
612
+ private expandQuery(userQuery: string): string[] {
613
+ const queries: string[] = [];
614
+ const seen = new Set<string>();
615
+
616
+ const addQuery = (q: string, clean = true) => {
617
+ // Clean up the query - remove trailing/leading punctuation and quotes
618
+ const cleaned = clean
619
+ ? q
620
+ .trim()
621
+ .replace(/^['".,]+|['".,]+$/g, '')
622
+ .trim()
623
+ : q;
624
+ const normalized = cleaned.toLowerCase();
625
+ // Skip very short queries (less than 10 chars) or duplicates
626
+ if (normalized.length >= 10 && !seen.has(normalized)) {
627
+ seen.add(normalized);
628
+ queries.push(cleaned);
629
+ }
630
+ };
631
+
632
+ // 1. Always include the full query (for overall context)
633
+ addQuery(userQuery, false);
634
+
635
+ // 2. Extract quoted text (single or double quotes)
636
+ const quotedMatches = userQuery.match(/["']([^"']+)["']/g);
637
+ if (quotedMatches) {
638
+ for (const match of quotedMatches) {
639
+ // Remove the quotes
640
+ const content = match.slice(1, -1);
641
+ addQuery(content);
642
+ }
643
+ }
644
+
645
+ // 3. Split by sentences and add each meaningful sentence
646
+ // Split on . ! ? but keep the delimiter for context
647
+ const sentences = userQuery.split(/(?<=[.!?])\s+/);
648
+ for (const sentence of sentences) {
649
+ const trimmed = sentence.trim();
650
+ // Only add if it's a substantial sentence (not just "Thanks!" etc.)
651
+ if (trimmed.length >= 20) {
652
+ addQuery(trimmed);
653
+ }
654
+ }
655
+
656
+ // 4. Extract key noun phrases using simple heuristics
657
+ // Look for patterns like "the X", "my X", "a X" where X is a noun phrase
658
+ const nounPhrasePatterns = [
659
+ /(?:the|my|a|an|your)\s+([a-z]+(?:\s+[a-z]+){0,3})/gi,
660
+ /(?:about|for|with|from)\s+([a-z]+(?:\s+[a-z]+){0,3})/gi,
661
+ ];
662
+
663
+ for (const pattern of nounPhrasePatterns) {
664
+ let match;
665
+ while ((match = pattern.exec(userQuery)) !== null) {
666
+ const phrase = match[1];
667
+ // Only add substantial phrases
668
+ if (phrase && phrase.length >= 5) {
669
+ addQuery(phrase);
670
+ }
671
+ }
672
+ }
673
+
674
+ // 5. Extract specific terms that might be important
675
+ // Look for capitalized words (proper nouns) that aren't at sentence start
676
+ const words = userQuery.split(/\s+/);
677
+ const properNouns: string[] = [];
678
+ for (let i = 1; i < words.length; i++) {
679
+ const word = words[i];
680
+ // Strip leading quotes for the capital check
681
+ const cleanWord = word.replace(/^['"]/, '');
682
+ // Check if word starts with capital and isn't after sentence-ending punctuation
683
+ if (cleanWord.length > 2 && /^[A-Z]/.test(cleanWord) && !/[.!?]$/.test(words[i - 1] || '')) {
684
+ // Collect consecutive capitalized words, stripping punctuation and quotes
685
+ const phrase = [cleanWord.replace(/[.,!?'"]+$/g, '')];
686
+ let j = i + 1;
687
+ while (j < words.length && /^['"]?[A-Z]/.test(words[j])) {
688
+ phrase.push(words[j].replace(/^['"]/, '').replace(/[.,!?'"]+$/g, ''));
689
+ j++;
690
+ }
691
+ if (phrase.join(' ').length >= 5) {
692
+ properNouns.push(phrase.join(' '));
693
+ }
694
+ }
695
+ }
696
+ for (const noun of properNouns) {
697
+ addQuery(noun);
698
+ }
699
+
700
+ console.log('[RAG Filter] Expanded query into', queries.length, 'sub-queries:', queries.slice(0, 5));
701
+
702
+ return queries;
703
+ }
704
+
705
+ /**
706
+ * Process input at each step - filter observations based on query similarity
707
+ */
708
+ async processInputStep(args: ProcessInputStepArgs) {
709
+ const { messages, messageList, state } = args;
710
+
711
+ console.log('[RAG Filter] processInputStep called, stepNumber:', args.stepNumber);
712
+
713
+ // Only process on step 0 (initial input)
714
+ // OM has already injected its full observations by this point
715
+ if (args.stepNumber !== 0) {
716
+ return messageList;
717
+ }
718
+
719
+ // Get OM's system messages directly from messageList
720
+ const omSystemMessages = messageList.getSystemMessages('observational-memory');
721
+ console.log('[RAG Filter] Found OM system messages:', omSystemMessages.length);
722
+ if (omSystemMessages.length === 0) {
723
+ // No observations to filter
724
+ return messageList;
725
+ }
726
+
727
+ const omSystemMessage = omSystemMessages[0];
728
+ const fullContext = typeof omSystemMessage.content === 'string' ? omSystemMessage.content : '';
729
+ console.log('[RAG Filter] Full context length:', fullContext.length);
730
+
731
+ // Extract the observations block
732
+ const observationsContent = extractObservationsBlock(fullContext);
733
+ console.log('[RAG Filter] Observations content length:', observationsContent?.length ?? 0);
734
+ if (!observationsContent) {
735
+ console.log('[RAG Filter] No observations block found, returning early');
736
+ return messageList;
737
+ }
738
+
739
+ // Get the user's query for similarity matching
740
+ // Debug: log all messages to understand the structure
741
+ console.log(
742
+ '[RAG Filter] Messages:',
743
+ messages.map(m => {
744
+ const content = m.content as unknown;
745
+ let preview = 'unknown';
746
+ if (typeof content === 'string') {
747
+ preview = (content as string).slice(0, 50);
748
+ } else if (Array.isArray(content)) {
749
+ preview = `array[${(content as unknown[]).length}]`;
750
+ }
751
+ return { role: m.role, contentType: typeof content, contentPreview: preview };
752
+ }),
753
+ );
754
+
755
+ const userQuery = getLatestUserMessage(messages);
756
+ console.log('[RAG Filter] User query found:', !!userQuery, 'messages count:', messages.length);
757
+ if (!userQuery) {
758
+ // No query to match against - keep full observations
759
+ console.log('[RAG Filter] No user query, returning early');
760
+ return messageList;
761
+ }
762
+
763
+ console.log('[RAG Filter] User query:', userQuery?.slice(0, 100));
764
+
765
+ // Index observations (will use cache for already-embedded content)
766
+ if (this.vectorStore.size === 0) {
767
+ console.log('[RAG Filter] Indexing observations...');
768
+ await this.indexObservations(observationsContent, state);
769
+ console.log('[RAG Filter] Indexed', this.vectorStore.size, 'chunks');
770
+ }
771
+
772
+ // Track all results and seen chunk IDs for deduplication
773
+ const seenChunkIds = new Set<string>();
774
+ let allResults: Array<{ chunk: ObservationChunk; similarity: number }> = [];
775
+ let mainQueryResultCount = 0;
776
+ let topSimilarity = 0;
777
+
778
+ // Determine queries to run
779
+ const queriesToRun = this.expandQueries ? this.expandQuery(userQuery) : [userQuery];
780
+
781
+ // Run each query and merge results
782
+ for (let i = 0; i < queriesToRun.length; i++) {
783
+ const query = queriesToRun[i];
784
+ const isMainQuery = i === 0;
785
+
786
+ const { embedding: queryEmbedding } = await embed({
787
+ model: this.embedder,
788
+ value: query,
789
+ });
790
+
791
+ // Use full topK for main query, smaller for expanded queries
792
+ const topK = isMainQuery ? this.topK : this.queryExpansionTopK;
793
+
794
+ const results = this.vectorStore.query(queryEmbedding, {
795
+ topK,
796
+ minSimilarity: isMainQuery ? this.minSimilarity : 0.8,
797
+ });
798
+
799
+ if (isMainQuery) {
800
+ mainQueryResultCount = results.length;
801
+ topSimilarity = results[0]?.similarity ?? 0;
802
+ console.log('[RAG Filter] Main query found', results.length, 'chunks');
803
+ console.log(
804
+ '[RAG Filter] Top 3 similarities:',
805
+ results
806
+ .slice(0, 3)
807
+ .map(r => r.similarity.toFixed(3))
808
+ .join(', '),
809
+ );
810
+ }
811
+
812
+ // Add unique results, keeping highest similarity for duplicates
813
+ for (const result of results) {
814
+ if (!seenChunkIds.has(result.chunk.id)) {
815
+ seenChunkIds.add(result.chunk.id);
816
+ allResults.push(result);
817
+ } else {
818
+ // Update similarity if this query found a higher score
819
+ const existing = allResults.find(r => r.chunk.id === result.chunk.id);
820
+ if (existing && result.similarity > existing.similarity) {
821
+ existing.similarity = result.similarity;
822
+ }
823
+ }
824
+ }
825
+ }
826
+
827
+ if (this.expandQueries && queriesToRun.length > 1) {
828
+ console.log(
829
+ '[RAG Filter] Query expansion added',
830
+ allResults.length - this.topK,
831
+ 'additional unique chunks from',
832
+ queriesToRun.length - 1,
833
+ 'sub-queries',
834
+ );
835
+ }
836
+ // Preference boost: run additional queries for user preferences
837
+ // This ensures critical user preferences are always included in context
838
+ if (this.preferenceBoost) {
839
+ const preferenceQueries = ['likes, prefers, wants, favourites'];
840
+ // const preferenceQueries = ['user prefers', 'user preference', 'user likes', 'user wants', "user's favorite"];
841
+ let preferenceHits = 0;
842
+
843
+ for (const prefQuery of preferenceQueries) {
844
+ const { embedding: prefEmbedding } = await embed({
845
+ model: this.embedder,
846
+ value: prefQuery,
847
+ });
848
+
849
+ const prefResults = this.vectorStore.query(prefEmbedding, {
850
+ topK: this.preferenceBoostTopK,
851
+ minSimilarity: this.preferenceBoostMinSimilarity,
852
+ });
853
+
854
+ // Add unique results (not already in main results or expanded queries)
855
+ for (const result of prefResults) {
856
+ if (!seenChunkIds.has(result.chunk.id) && result.similarity > 0.9) {
857
+ seenChunkIds.add(result.chunk.id);
858
+ allResults.push(result);
859
+ preferenceHits++;
860
+ }
861
+ }
862
+ }
863
+
864
+ console.log('[RAG Filter] Preference boost added', preferenceHits, 'unique chunks');
865
+ }
866
+
867
+ // Expand results based on similarity score (high score = more context)
868
+ const expandedChunks = this.expandResults(allResults);
869
+
870
+ // Build filtered observations with preserved structure (thread + date grouping)
871
+ const filteredObservations = rebuildObservationsWithStructure(expandedChunks);
872
+
873
+ // Build the new context with filtered observations
874
+ let newContext = `
875
+ The following observations block contains your memory of past conversations with this user. Use these observations to provide personalized, contextually relevant responses.
876
+
877
+ <observations>
878
+ ${filteredObservations}
879
+ </observations>`;
880
+
881
+ // Optionally include patterns (already compressed, usually worth including)
882
+ if (this.includePatterns) {
883
+ const patterns = extractPatternsBlock(fullContext);
884
+ if (patterns) {
885
+ newContext += `\n\n<patterns>\n${patterns}\n</patterns>`;
886
+ }
887
+ }
888
+
889
+ // Optionally include current-task
890
+ if (this.includeCurrentTask) {
891
+ const currentTask = extractCurrentTaskBlock(fullContext);
892
+ if (currentTask) {
893
+ newContext += `\n\n<current-task>\n${currentTask}\n</current-task>`;
894
+ }
895
+ }
896
+
897
+ // Optionally include suggested-response
898
+ if (this.includeSuggestedResponse) {
899
+ const suggestedResponse = extractSuggestedResponseBlock(fullContext);
900
+ if (suggestedResponse) {
901
+ newContext += `\n\n<suggested-response>\n${suggestedResponse}\n</suggested-response>`;
902
+ }
903
+ }
904
+
905
+ // Replace OM's system message with our filtered version
906
+ // Get all system messages, replace OM's, and set them all back
907
+ const allSystemMessages = messageList.getAllSystemMessages();
908
+ const updatedSystemMessages = allSystemMessages.map(msg => {
909
+ // Replace the OM message with our filtered version
910
+ if (msg === omSystemMessage) {
911
+ return {
912
+ role: 'system' as const,
913
+ content: newContext,
914
+ };
915
+ }
916
+ return msg;
917
+ });
918
+ messageList.replaceAllSystemMessages(updatedSystemMessages);
919
+ console.log('[RAG Filter] Replaced system messages. New context length:', newContext.length, 'chars');
920
+
921
+ // Store stats in state for debugging
922
+ state.ragStats = {
923
+ originalLines: this.allParsedObservations.length,
924
+ matchedLines: mainQueryResultCount,
925
+ expandedQueryCount: queriesToRun.length,
926
+ totalMatches: allResults.length,
927
+ expandedLines: expandedChunks.length,
928
+ topSimilarity,
929
+ query: userQuery.slice(0, 100),
930
+ };
931
+
932
+ return messageList;
933
+ }
934
+
935
+ /**
936
+ * Index observations into the vector store with content-based caching
937
+ */
938
+ private async indexObservations(observations: string, state: Record<string, unknown>): Promise<void> {
939
+ const parsedObs = parseObservationLines(observations);
940
+ if (parsedObs.length === 0) return;
941
+
942
+ // Store all parsed observations for neighbor lookup during expansion
943
+ this.allParsedObservations = parsedObs;
944
+
945
+ // Get resourceId from state if available
946
+ const resourceId = state.resourceId as string | undefined;
947
+
948
+ // Separate cached vs uncached observations
949
+ const uncachedObs: ParsedObservation[] = [];
950
+ const cachedChunks: ObservationChunk[] = [];
951
+ const obsHashes: string[] = [];
952
+
953
+ for (const obs of parsedObs) {
954
+ const hash = await this.hashContent(obs.line);
955
+ obsHashes.push(hash);
956
+
957
+ const cachedEmbedding = this.getCachedEmbedding(hash);
958
+ if (cachedEmbedding) {
959
+ // Use cached embedding
960
+ cachedChunks.push({
961
+ id: `obs_${hash}`,
962
+ embedding: cachedEmbedding,
963
+ metadata: {
964
+ threadId: obs.threadId,
965
+ resourceId,
966
+ dateGroup: obs.dateGroup,
967
+ time: obs.time,
968
+ labels: obs.labels,
969
+ lineIndex: obs.lineIndex,
970
+ content: obs.line,
971
+ },
972
+ });
973
+ } else {
974
+ uncachedObs.push(obs);
975
+ }
976
+ }
977
+
978
+ console.log(`[RAG Filter] Cache: ${cachedChunks.length} hits, ${uncachedObs.length} misses`);
979
+
980
+ // Embed uncached observations in batches
981
+ if (uncachedObs.length > 0) {
982
+ const batchSize = 25;
983
+ for (let b = 0; b < uncachedObs.length; b += batchSize) {
984
+ const batch = uncachedObs.slice(b, b + batchSize);
985
+
986
+ const { embeddings } = await embedMany({
987
+ model: this.embedder,
988
+ values: batch.map(l => l.line),
989
+ });
990
+
991
+ // Create chunks and save to cache
992
+ for (let i = 0; i < batch.length; i++) {
993
+ const obs = batch[i];
994
+ const hash = await this.hashContent(obs.line);
995
+ const embedding = embeddings[i];
996
+
997
+ // Save to cache
998
+ this.saveEmbeddingToCache(hash, embedding);
999
+
1000
+ cachedChunks.push({
1001
+ id: `obs_${hash}`,
1002
+ embedding,
1003
+ metadata: {
1004
+ threadId: obs.threadId,
1005
+ resourceId,
1006
+ dateGroup: obs.dateGroup,
1007
+ time: obs.time,
1008
+ labels: obs.labels,
1009
+ lineIndex: obs.lineIndex,
1010
+ content: obs.line,
1011
+ },
1012
+ });
1013
+ }
1014
+ }
1015
+ }
1016
+
1017
+ this.vectorStore.upsert(cachedChunks);
1018
+ }
1019
+
1020
+ /**
1021
+ * Expand search results by including neighboring observations based on similarity score.
1022
+ * High score (≥0.6): Include ±2 lines of context
1023
+ * Medium score (0.4-0.6): Include ±1 line of context
1024
+ * Low score (<0.4): Just the matched line
1025
+ */
1026
+ private expandResults(
1027
+ results: Array<{ chunk: ObservationChunk; similarity: number }>,
1028
+ ): Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }> {
1029
+ if (!this.expandContext || this.allParsedObservations.length === 0) {
1030
+ // Return results as-is without expansion
1031
+ return results.map(r => r.chunk);
1032
+ }
1033
+
1034
+ // Create a lookup for all observations by threadId and lineIndex
1035
+ const obsLookup = new Map<string, ParsedObservation[]>();
1036
+ for (const obs of this.allParsedObservations) {
1037
+ const key = obs.threadId;
1038
+ if (!obsLookup.has(key)) {
1039
+ obsLookup.set(key, []);
1040
+ }
1041
+ obsLookup.get(key)!.push(obs);
1042
+ }
1043
+
1044
+ // Sort each thread's observations by lineIndex
1045
+ for (const [, obsList] of Array.from(obsLookup.entries())) {
1046
+ obsList.sort((a, b) => a.lineIndex - b.lineIndex);
1047
+ }
1048
+
1049
+ // Track which lines we've already included (by threadId + lineIndex)
1050
+ const includedLines = new Set<string>();
1051
+ const expandedChunks: Array<{ metadata: ObservationChunkMetadata & { threadId: string; dateGroup: string } }> = [];
1052
+
1053
+ for (const result of results) {
1054
+ const { chunk, similarity } = result;
1055
+ const { threadId, lineIndex, resourceId } = chunk.metadata;
1056
+
1057
+ // Determine expansion range based on score
1058
+ // High score = include FULL thread, medium = ±1 line, low = just the match
1059
+ const includeFullThread = similarity >= this.highScoreThreshold;
1060
+ let range = 0;
1061
+ if (!includeFullThread && similarity >= this.mediumScoreThreshold) {
1062
+ range = 2; // ±1 line for medium confidence
1063
+ }
1064
+ // range = 0 for low confidence (just the match)
1065
+
1066
+ // Get observations for this thread
1067
+ const threadObs = obsLookup.get(threadId) || [];
1068
+ if (threadObs.length === 0) {
1069
+ // Fallback: just include the original chunk
1070
+ const lineKey = `${threadId}:${lineIndex}`;
1071
+ if (!includedLines.has(lineKey)) {
1072
+ includedLines.add(lineKey);
1073
+ expandedChunks.push(chunk);
1074
+ }
1075
+ continue;
1076
+ }
1077
+
1078
+ // Find observations within range (or all if includeFullThread)
1079
+ for (const obs of threadObs) {
1080
+ const inRange = includeFullThread || (obs.lineIndex >= lineIndex - range && obs.lineIndex <= lineIndex + range);
1081
+ if (inRange) {
1082
+ const lineKey = `${threadId}:${obs.lineIndex}`;
1083
+ if (!includedLines.has(lineKey)) {
1084
+ includedLines.add(lineKey);
1085
+ expandedChunks.push({
1086
+ metadata: {
1087
+ threadId: obs.threadId,
1088
+ resourceId,
1089
+ dateGroup: obs.dateGroup,
1090
+ time: obs.time,
1091
+ labels: obs.labels,
1092
+ lineIndex: obs.lineIndex,
1093
+ content: obs.line,
1094
+ },
1095
+ });
1096
+ }
1097
+ }
1098
+ }
1099
+ }
1100
+
1101
+ // Sort expanded chunks by threadId, dateGroup, then lineIndex for proper reconstruction
1102
+ expandedChunks.sort((a, b) => {
1103
+ if (a.metadata.threadId !== b.metadata.threadId) {
1104
+ return a.metadata.threadId.localeCompare(b.metadata.threadId);
1105
+ }
1106
+ if (a.metadata.dateGroup !== b.metadata.dateGroup) {
1107
+ return a.metadata.dateGroup.localeCompare(b.metadata.dateGroup);
1108
+ }
1109
+ return a.metadata.lineIndex - b.metadata.lineIndex;
1110
+ });
1111
+
1112
+ console.log(
1113
+ `[RAG Filter] Context expansion: ${results.length} matches → ${expandedChunks.length} lines (range based on score)`,
1114
+ );
1115
+
1116
+ return expandedChunks;
1117
+ }
1118
+
1119
+ /**
1120
+ * Process output - index new observations after OM creates them
1121
+ *
1122
+ * Note: This is called after OM's processOutputResult, so new observations
1123
+ * should be available in the OM record. For now, we'll skip this and
1124
+ * rely on indexing during processInputStep.
1125
+ */
1126
+ async processOutputResult(args: ProcessOutputResultArgs) {
1127
+ // For the initial implementation, we index observations lazily during
1128
+ // processInputStep. This avoids the complexity of coordinating with OM's
1129
+ // output processing.
1130
+ //
1131
+ // Future enhancement: Index new observations here for better performance
1132
+ // on subsequent queries within the same session.
1133
+ return args.messageList;
1134
+ }
1135
+ }
1136
+
1137
+ /**
1138
+ * Create an ObservationSemanticFilter processor
1139
+ */
1140
+ export function createObservationSemanticFilter(config: ObservationSemanticFilterConfig): ObservationSemanticFilter {
1141
+ return new ObservationSemanticFilter(config);
1142
+ }