@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,1128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding-Based Relationship Detector
|
|
3
|
+
*
|
|
4
|
+
* Main detector class that orchestrates relationship detection between memories
|
|
5
|
+
* using vector similarity, temporal analysis, entity overlap, and optional LLM verification.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import type { RelationshipType, Entity } from '../../types/index.js'
|
|
9
|
+
import type { Memory, Relationship } from '../memory.types.js'
|
|
10
|
+
import type { EmbeddingService } from '../embedding.service.js'
|
|
11
|
+
import { cosineSimilarity } from '../embedding.service.js'
|
|
12
|
+
import { generateId } from '../../utils/id.js'
|
|
13
|
+
import { getLogger } from '../../utils/logger.js'
|
|
14
|
+
import { AppError, ErrorCode } from '../../utils/errors.js'
|
|
15
|
+
import type {
|
|
16
|
+
RelationshipConfig,
|
|
17
|
+
RelationshipCandidate,
|
|
18
|
+
RelationshipDetectionResult,
|
|
19
|
+
Contradiction,
|
|
20
|
+
ContradictionType,
|
|
21
|
+
ContradictionResolution,
|
|
22
|
+
VectorStore,
|
|
23
|
+
VectorSearchResult,
|
|
24
|
+
LLMProvider,
|
|
25
|
+
DetectedRelationship,
|
|
26
|
+
RelationshipDetectionStats,
|
|
27
|
+
CachedRelationshipScore,
|
|
28
|
+
DetectionStrategyType,
|
|
29
|
+
} from './types.js'
|
|
30
|
+
import { DEFAULT_RELATIONSHIP_CONFIG, generateCacheKey } from './types.js'
|
|
31
|
+
|
|
32
|
+
const logger = getLogger('EmbeddingRelationshipDetector')
|
|
33
|
+
|
|
34
|
+
// ============================================================================
|
|
35
|
+
// Embedding Helper (candidate list)
|
|
36
|
+
// ============================================================================
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Detect relationships using embeddings from a provided candidate list.
|
|
40
|
+
* This helper is useful when you already have candidate memories
|
|
41
|
+
* and want a single-pass relationship detection result.
|
|
42
|
+
*/
|
|
43
|
+
export async function detectRelationshipsWithEmbeddings(
|
|
44
|
+
newMemory: Memory,
|
|
45
|
+
candidates: Memory[],
|
|
46
|
+
embeddingService: EmbeddingService,
|
|
47
|
+
options: {
|
|
48
|
+
containerTag?: string
|
|
49
|
+
config?: Partial<RelationshipConfig>
|
|
50
|
+
} = {}
|
|
51
|
+
): Promise<RelationshipDetectionResult> {
|
|
52
|
+
const vectorStore = new InMemoryVectorStoreAdapter()
|
|
53
|
+
|
|
54
|
+
if (candidates.length > 0) {
|
|
55
|
+
const embeddings = await embeddingService.batchEmbed(candidates.map((m) => m.content))
|
|
56
|
+
for (let i = 0; i < candidates.length; i++) {
|
|
57
|
+
const candidate = candidates[i]
|
|
58
|
+
const embedding = embeddings[i]
|
|
59
|
+
if (candidate && embedding) {
|
|
60
|
+
candidate.embedding = embedding
|
|
61
|
+
vectorStore.addMemory(candidate, embedding)
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
if (!newMemory.embedding || newMemory.embedding.length === 0) {
|
|
67
|
+
newMemory.embedding = await embeddingService.generateEmbedding(newMemory.content)
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const detector = new EmbeddingRelationshipDetector(embeddingService, vectorStore, options.config)
|
|
71
|
+
|
|
72
|
+
return detector.detectRelationships(newMemory, {
|
|
73
|
+
containerTag: options.containerTag,
|
|
74
|
+
excludeIds: [newMemory.id],
|
|
75
|
+
})
|
|
76
|
+
}
|
|
77
|
+
|
|
78
|
+
// ============================================================================
|
|
79
|
+
// Helper Functions (from strategies.ts)
|
|
80
|
+
// ============================================================================
|
|
81
|
+
|
|
82
|
+
/**
|
|
83
|
+
* Create a detected relationship object
|
|
84
|
+
*/
|
|
85
|
+
function createDetectedRelationship(
|
|
86
|
+
sourceMemory: Memory,
|
|
87
|
+
targetMemory: Memory,
|
|
88
|
+
type: RelationshipType,
|
|
89
|
+
candidate: RelationshipCandidate,
|
|
90
|
+
strategyName: string,
|
|
91
|
+
llmVerified: boolean = false,
|
|
92
|
+
llmConfidence?: number
|
|
93
|
+
): DetectedRelationship {
|
|
94
|
+
const relationship: Relationship = {
|
|
95
|
+
id: generateId(),
|
|
96
|
+
sourceMemoryId: sourceMemory.id,
|
|
97
|
+
targetMemoryId: targetMemory.id,
|
|
98
|
+
type,
|
|
99
|
+
confidence: candidate.combinedScore,
|
|
100
|
+
description: `${type} relationship detected via ${strategyName} strategy`,
|
|
101
|
+
createdAt: new Date(),
|
|
102
|
+
metadata: {
|
|
103
|
+
vectorSimilarity: candidate.vectorSimilarity,
|
|
104
|
+
entityOverlap: candidate.entityOverlap,
|
|
105
|
+
temporalScore: candidate.temporalScore,
|
|
106
|
+
detectionStrategy: strategyName,
|
|
107
|
+
},
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Validate strategy name is a valid DetectionStrategyType
|
|
111
|
+
const validStrategy: DetectionStrategyType =
|
|
112
|
+
strategyName === 'similarity' ||
|
|
113
|
+
strategyName === 'temporal' ||
|
|
114
|
+
strategyName === 'entityOverlap' ||
|
|
115
|
+
strategyName === 'llmVerification' ||
|
|
116
|
+
strategyName === 'hybrid'
|
|
117
|
+
? strategyName
|
|
118
|
+
: 'hybrid'
|
|
119
|
+
|
|
120
|
+
return {
|
|
121
|
+
relationship,
|
|
122
|
+
score: candidate.combinedScore,
|
|
123
|
+
vectorSimilarity: candidate.vectorSimilarity,
|
|
124
|
+
entityOverlap: candidate.entityOverlap,
|
|
125
|
+
temporalScore: candidate.temporalScore,
|
|
126
|
+
llmVerified,
|
|
127
|
+
llmConfidence,
|
|
128
|
+
detectionStrategy: validStrategy,
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
/**
|
|
133
|
+
* Check if content contains update/correction indicators
|
|
134
|
+
*/
|
|
135
|
+
function hasUpdateIndicators(content: string): boolean {
|
|
136
|
+
const patterns = [
|
|
137
|
+
/\b(?:update|updated|updating|correction|corrected)\b/i,
|
|
138
|
+
/\b(?:now|actually|instead)\b/i,
|
|
139
|
+
/\b(?:changed|revised|modified)\b/i,
|
|
140
|
+
/\b(?:no longer|used to be|previously)\b/i,
|
|
141
|
+
]
|
|
142
|
+
return patterns.some((p) => p.test(content))
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* Check if content contains extension indicators
|
|
147
|
+
*/
|
|
148
|
+
function hasExtensionIndicators(content: string): boolean {
|
|
149
|
+
const patterns = [
|
|
150
|
+
/\b(?:also|additionally|furthermore|moreover)\b/i,
|
|
151
|
+
/\b(?:in addition|on top of|besides)\b/i,
|
|
152
|
+
/\b(?:extending|building on|adding to)\b/i,
|
|
153
|
+
]
|
|
154
|
+
return patterns.some((p) => p.test(content))
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
/**
|
|
158
|
+
* Check if content contains contradiction indicators
|
|
159
|
+
*/
|
|
160
|
+
function hasContradictionIndicators(content: string): boolean {
|
|
161
|
+
const patterns = [
|
|
162
|
+
/\b(?:however|but|although|despite)\b/i,
|
|
163
|
+
/\b(?:contrary|opposite|different)\b/i,
|
|
164
|
+
/\b(?:not true|incorrect|wrong|false)\b/i,
|
|
165
|
+
/\b(?:disagree|dispute|reject)\b/i,
|
|
166
|
+
]
|
|
167
|
+
return patterns.some((p) => p.test(content))
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
/**
|
|
171
|
+
* Check if content contains supersession indicators
|
|
172
|
+
*/
|
|
173
|
+
function hasSupersessionIndicators(content: string): boolean {
|
|
174
|
+
const patterns = [
|
|
175
|
+
/\b(?:replaces|supersedes|overrides)\b/i,
|
|
176
|
+
/\b(?:no longer|obsolete|deprecated)\b/i,
|
|
177
|
+
/\b(?:new version|latest|current)\b/i,
|
|
178
|
+
]
|
|
179
|
+
return patterns.some((p) => p.test(content))
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Check if content contains causal/derivation indicators
|
|
184
|
+
*/
|
|
185
|
+
function hasCausalIndicators(content: string): boolean {
|
|
186
|
+
const patterns = [
|
|
187
|
+
/\b(?:therefore|thus|hence|consequently)\b/i,
|
|
188
|
+
/\b(?:because|since|as a result)\b/i,
|
|
189
|
+
/\b(?:based on|derived from|follows from)\b/i,
|
|
190
|
+
/\b(?:leads to|results in|causes)\b/i,
|
|
191
|
+
]
|
|
192
|
+
return patterns.some((p) => p.test(content))
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
// ============================================================================
|
|
196
|
+
// In-Memory Vector Store Adapter
|
|
197
|
+
// ============================================================================
|
|
198
|
+
|
|
199
|
+
/**
|
|
200
|
+
* Simple in-memory vector store adapter for relationship detection.
|
|
201
|
+
* Can be replaced with a proper vector database in production.
|
|
202
|
+
*/
|
|
203
|
+
export class InMemoryVectorStoreAdapter implements VectorStore {
|
|
204
|
+
private entries: Map<string, { memory: Memory; embedding: number[] }> = new Map()
|
|
205
|
+
|
|
206
|
+
/**
|
|
207
|
+
* Add a memory with its embedding
|
|
208
|
+
*/
|
|
209
|
+
addMemory(memory: Memory, embedding: number[]): void {
|
|
210
|
+
this.entries.set(memory.id, { memory, embedding })
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Add multiple memories with their embeddings
|
|
215
|
+
*/
|
|
216
|
+
addMemories(items: Array<{ memory: Memory; embedding: number[] }>): void {
|
|
217
|
+
for (const item of items) {
|
|
218
|
+
this.entries.set(item.memory.id, item)
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
/**
|
|
223
|
+
* Remove a memory
|
|
224
|
+
*/
|
|
225
|
+
removeMemory(memoryId: string): boolean {
|
|
226
|
+
return this.entries.delete(memoryId)
|
|
227
|
+
}
|
|
228
|
+
|
|
229
|
+
/**
|
|
230
|
+
* Update a memory's embedding
|
|
231
|
+
*/
|
|
232
|
+
updateEmbedding(memoryId: string, embedding: number[]): boolean {
|
|
233
|
+
const entry = this.entries.get(memoryId)
|
|
234
|
+
if (entry) {
|
|
235
|
+
entry.embedding = embedding
|
|
236
|
+
return true
|
|
237
|
+
}
|
|
238
|
+
return false
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Get all memories
|
|
243
|
+
*/
|
|
244
|
+
getAllMemories(): Memory[] {
|
|
245
|
+
return Array.from(this.entries.values()).map((e) => e.memory)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Clear all entries
|
|
250
|
+
*/
|
|
251
|
+
clear(): void {
|
|
252
|
+
this.entries.clear()
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
async findSimilar(
|
|
256
|
+
embedding: number[],
|
|
257
|
+
limit: number,
|
|
258
|
+
threshold: number,
|
|
259
|
+
filters?: { containerTag?: string; excludeIds?: string[] }
|
|
260
|
+
): Promise<VectorSearchResult[]> {
|
|
261
|
+
const results: VectorSearchResult[] = []
|
|
262
|
+
const excludeSet = new Set(filters?.excludeIds || [])
|
|
263
|
+
|
|
264
|
+
for (const [id, entry] of this.entries) {
|
|
265
|
+
if (excludeSet.has(id)) continue
|
|
266
|
+
if (filters?.containerTag && entry.memory.containerTag !== filters.containerTag) continue
|
|
267
|
+
|
|
268
|
+
const similarity = cosineSimilarity(embedding, entry.embedding)
|
|
269
|
+
if (similarity >= threshold) {
|
|
270
|
+
results.push({
|
|
271
|
+
memoryId: id,
|
|
272
|
+
memory: entry.memory,
|
|
273
|
+
similarity,
|
|
274
|
+
})
|
|
275
|
+
}
|
|
276
|
+
}
|
|
277
|
+
|
|
278
|
+
// Sort by similarity descending and limit
|
|
279
|
+
results.sort((a, b) => b.similarity - a.similarity)
|
|
280
|
+
return results.slice(0, limit)
|
|
281
|
+
}
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
// ============================================================================
|
|
285
|
+
// Embedding Relationship Detector
|
|
286
|
+
// ============================================================================
|
|
287
|
+
|
|
288
|
+
/**
|
|
289
|
+
* Embedding-based relationship detector.
|
|
290
|
+
* Uses vector similarity and configurable strategies to detect relationships
|
|
291
|
+
* between memories.
|
|
292
|
+
*/
|
|
293
|
+
export class EmbeddingRelationshipDetector {
|
|
294
|
+
private readonly embeddingService: EmbeddingService
|
|
295
|
+
private readonly vectorStore: VectorStore
|
|
296
|
+
private readonly config: RelationshipConfig
|
|
297
|
+
private readonly cache: Map<string, CachedRelationshipScore>
|
|
298
|
+
private llmProvider?: LLMProvider
|
|
299
|
+
|
|
300
|
+
constructor(
|
|
301
|
+
embeddingService: EmbeddingService,
|
|
302
|
+
vectorStore: VectorStore,
|
|
303
|
+
config: Partial<RelationshipConfig> = {},
|
|
304
|
+
llmProvider?: LLMProvider
|
|
305
|
+
) {
|
|
306
|
+
this.embeddingService = embeddingService
|
|
307
|
+
this.vectorStore = vectorStore
|
|
308
|
+
this.config = { ...DEFAULT_RELATIONSHIP_CONFIG, ...config }
|
|
309
|
+
this.llmProvider = llmProvider
|
|
310
|
+
this.cache = new Map()
|
|
311
|
+
|
|
312
|
+
logger.debug('EmbeddingRelationshipDetector initialized', {
|
|
313
|
+
config: this.config,
|
|
314
|
+
hasLLMProvider: !!llmProvider,
|
|
315
|
+
})
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
// ============================================================================
|
|
319
|
+
// Private Detection Methods
|
|
320
|
+
// ============================================================================
|
|
321
|
+
|
|
322
|
+
/**
|
|
323
|
+
* Detect relationships using vector similarity thresholds (from SimilarityStrategy)
|
|
324
|
+
*/
|
|
325
|
+
private async detectBySimilarity(
|
|
326
|
+
newMemory: Memory,
|
|
327
|
+
candidates: RelationshipCandidate[]
|
|
328
|
+
): Promise<DetectedRelationship[]> {
|
|
329
|
+
const relationships: DetectedRelationship[] = []
|
|
330
|
+
const { thresholds } = this.config
|
|
331
|
+
|
|
332
|
+
for (const candidate of candidates) {
|
|
333
|
+
const sim = candidate.vectorSimilarity
|
|
334
|
+
let detectedType: RelationshipType | null = null
|
|
335
|
+
let adjustedConfidence = sim
|
|
336
|
+
|
|
337
|
+
// Check for supersedes (highest threshold)
|
|
338
|
+
if (sim >= thresholds.supersedes) {
|
|
339
|
+
if (hasSupersessionIndicators(newMemory.content)) {
|
|
340
|
+
detectedType = 'supersedes'
|
|
341
|
+
adjustedConfidence = Math.min(sim + 0.05, 1.0)
|
|
342
|
+
} else if (hasUpdateIndicators(newMemory.content)) {
|
|
343
|
+
detectedType = 'updates'
|
|
344
|
+
}
|
|
345
|
+
}
|
|
346
|
+
// Check for updates
|
|
347
|
+
else if (sim >= thresholds.updates) {
|
|
348
|
+
if (hasUpdateIndicators(newMemory.content)) {
|
|
349
|
+
detectedType = 'updates'
|
|
350
|
+
adjustedConfidence = Math.min(sim + 0.05, 1.0)
|
|
351
|
+
} else if (hasContradictionIndicators(newMemory.content) && this.config.enableContradictionDetection) {
|
|
352
|
+
detectedType = 'contradicts'
|
|
353
|
+
}
|
|
354
|
+
}
|
|
355
|
+
// Check for contradicts
|
|
356
|
+
else if (sim >= thresholds.contradicts && this.config.enableContradictionDetection) {
|
|
357
|
+
if (hasContradictionIndicators(newMemory.content)) {
|
|
358
|
+
detectedType = 'contradicts'
|
|
359
|
+
}
|
|
360
|
+
}
|
|
361
|
+
// Check for extends
|
|
362
|
+
else if (sim >= thresholds.extends) {
|
|
363
|
+
if (hasExtensionIndicators(newMemory.content)) {
|
|
364
|
+
detectedType = 'extends'
|
|
365
|
+
adjustedConfidence = Math.min(sim + 0.05, 1.0)
|
|
366
|
+
} else {
|
|
367
|
+
// High similarity but no explicit indicator - mark as related
|
|
368
|
+
detectedType = 'related'
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
// Check for derives
|
|
372
|
+
else if (sim >= thresholds.derives && this.config.enableCausalDetection) {
|
|
373
|
+
if (hasCausalIndicators(newMemory.content)) {
|
|
374
|
+
detectedType = 'derives'
|
|
375
|
+
}
|
|
376
|
+
}
|
|
377
|
+
// Check for related (lowest threshold)
|
|
378
|
+
else if (sim >= thresholds.related) {
|
|
379
|
+
detectedType = 'related'
|
|
380
|
+
}
|
|
381
|
+
|
|
382
|
+
if (detectedType) {
|
|
383
|
+
// Update combined score with adjusted confidence
|
|
384
|
+
const adjustedCandidate: RelationshipCandidate = {
|
|
385
|
+
...candidate,
|
|
386
|
+
combinedScore: adjustedConfidence,
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
relationships.push(
|
|
390
|
+
createDetectedRelationship(newMemory, candidate.memory, detectedType, adjustedCandidate, 'similarity')
|
|
391
|
+
)
|
|
392
|
+
}
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
return relationships
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Detect relationships using temporal proximity (from TemporalStrategy)
|
|
400
|
+
*/
|
|
401
|
+
private async detectByTemporal(
|
|
402
|
+
newMemory: Memory,
|
|
403
|
+
candidates: RelationshipCandidate[]
|
|
404
|
+
): Promise<DetectedRelationship[]> {
|
|
405
|
+
const relationships: DetectedRelationship[] = []
|
|
406
|
+
|
|
407
|
+
// Only process candidates with moderate similarity
|
|
408
|
+
const relevantCandidates = candidates.filter((c) => c.vectorSimilarity >= this.config.thresholds.related * 0.8)
|
|
409
|
+
|
|
410
|
+
for (const candidate of relevantCandidates) {
|
|
411
|
+
const timeDiff = Math.abs(newMemory.createdAt.getTime() - candidate.memory.createdAt.getTime())
|
|
412
|
+
const oneHour = 60 * 60 * 1000
|
|
413
|
+
const oneDay = 24 * oneHour
|
|
414
|
+
|
|
415
|
+
// Check for rapid succession updates (within 1 hour)
|
|
416
|
+
if (timeDiff < oneHour && candidate.vectorSimilarity >= 0.75) {
|
|
417
|
+
// New memory likely updates the old one
|
|
418
|
+
const isNewer = newMemory.createdAt > candidate.memory.createdAt
|
|
419
|
+
if (isNewer && candidate.memory.type === newMemory.type) {
|
|
420
|
+
// Boost temporal score
|
|
421
|
+
const adjustedCandidate: RelationshipCandidate = {
|
|
422
|
+
...candidate,
|
|
423
|
+
temporalScore: 0.95,
|
|
424
|
+
combinedScore: Math.min(candidate.vectorSimilarity * 0.7 + 0.3, 1.0),
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
relationships.push(
|
|
428
|
+
createDetectedRelationship(newMemory, candidate.memory, 'updates', adjustedCandidate, 'temporal')
|
|
429
|
+
)
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
// Check for related context (within same day)
|
|
433
|
+
else if (timeDiff < oneDay && candidate.vectorSimilarity >= 0.6) {
|
|
434
|
+
// Context-related memories from the same session/day
|
|
435
|
+
if (candidate.memory.containerTag === newMemory.containerTag) {
|
|
436
|
+
const adjustedCandidate: RelationshipCandidate = {
|
|
437
|
+
...candidate,
|
|
438
|
+
temporalScore: 0.8,
|
|
439
|
+
combinedScore: Math.min(candidate.vectorSimilarity * 0.8 + 0.1, 1.0),
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
relationships.push(
|
|
443
|
+
createDetectedRelationship(newMemory, candidate.memory, 'related', adjustedCandidate, 'temporal')
|
|
444
|
+
)
|
|
445
|
+
}
|
|
446
|
+
}
|
|
447
|
+
}
|
|
448
|
+
|
|
449
|
+
return relationships
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/**
|
|
453
|
+
* Type guard to check if an entity is valid
|
|
454
|
+
*/
|
|
455
|
+
private isValidEntity(entity: unknown): entity is Entity {
|
|
456
|
+
return (
|
|
457
|
+
typeof entity === 'object' && entity !== null && 'name' in entity && typeof (entity as Entity).name === 'string'
|
|
458
|
+
)
|
|
459
|
+
}
|
|
460
|
+
|
|
461
|
+
/**
|
|
462
|
+
* Detect relationships using entity overlap (from EntityOverlapStrategy)
|
|
463
|
+
*/
|
|
464
|
+
private async detectByEntityOverlap(
|
|
465
|
+
newMemory: Memory,
|
|
466
|
+
candidates: RelationshipCandidate[]
|
|
467
|
+
): Promise<DetectedRelationship[]> {
|
|
468
|
+
const relationships: DetectedRelationship[] = []
|
|
469
|
+
|
|
470
|
+
const rawEntities = newMemory.metadata?.entities
|
|
471
|
+
const newEntities = Array.isArray(rawEntities)
|
|
472
|
+
? (rawEntities.filter(this.isValidEntity.bind(this)) as Entity[])
|
|
473
|
+
: []
|
|
474
|
+
|
|
475
|
+
if (newEntities.length === 0) {
|
|
476
|
+
return relationships
|
|
477
|
+
}
|
|
478
|
+
|
|
479
|
+
for (const candidate of candidates) {
|
|
480
|
+
const rawCandidateEntities = candidate.memory.metadata?.entities
|
|
481
|
+
const candidateEntities = Array.isArray(rawCandidateEntities)
|
|
482
|
+
? (rawCandidateEntities.filter(this.isValidEntity.bind(this)) as Entity[])
|
|
483
|
+
: []
|
|
484
|
+
|
|
485
|
+
if (candidateEntities.length === 0) continue
|
|
486
|
+
|
|
487
|
+
// Calculate entity overlap
|
|
488
|
+
const names1 = new Set(newEntities.map((e) => e.name.toLowerCase()))
|
|
489
|
+
const names2 = new Set(candidateEntities.map((e) => e.name.toLowerCase()))
|
|
490
|
+
|
|
491
|
+
const intersection = [...names1].filter((n) => names2.has(n)).length
|
|
492
|
+
const union = new Set([...names1, ...names2]).size
|
|
493
|
+
const entityOverlap = union > 0 ? intersection / union : 0
|
|
494
|
+
|
|
495
|
+
// Significant entity overlap (>50%) suggests strong relationship
|
|
496
|
+
if (entityOverlap >= 0.5) {
|
|
497
|
+
// Combine with vector similarity for relationship type
|
|
498
|
+
const combinedScore =
|
|
499
|
+
candidate.vectorSimilarity * (1 - this.config.entityOverlapWeight) +
|
|
500
|
+
entityOverlap * this.config.entityOverlapWeight
|
|
501
|
+
|
|
502
|
+
let relationshipType: RelationshipType = 'related'
|
|
503
|
+
|
|
504
|
+
// High entity overlap + high similarity = likely update or extension
|
|
505
|
+
if (entityOverlap >= 0.8 && candidate.vectorSimilarity >= 0.7) {
|
|
506
|
+
if (hasUpdateIndicators(newMemory.content)) {
|
|
507
|
+
relationshipType = 'updates'
|
|
508
|
+
} else if (hasExtensionIndicators(newMemory.content)) {
|
|
509
|
+
relationshipType = 'extends'
|
|
510
|
+
}
|
|
511
|
+
}
|
|
512
|
+
|
|
513
|
+
const adjustedCandidate: RelationshipCandidate = {
|
|
514
|
+
...candidate,
|
|
515
|
+
entityOverlap,
|
|
516
|
+
combinedScore: Math.min(combinedScore, 1.0),
|
|
517
|
+
}
|
|
518
|
+
|
|
519
|
+
relationships.push(
|
|
520
|
+
createDetectedRelationship(newMemory, candidate.memory, relationshipType, adjustedCandidate, 'entityOverlap')
|
|
521
|
+
)
|
|
522
|
+
}
|
|
523
|
+
}
|
|
524
|
+
|
|
525
|
+
return relationships
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Merge relationships from multiple detection approaches (from HybridStrategy)
|
|
530
|
+
*/
|
|
531
|
+
private mergeRelationships(allRelationships: DetectedRelationship[]): DetectedRelationship[] {
|
|
532
|
+
// Merge results, keeping highest confidence per relationship pair
|
|
533
|
+
const relationshipMap = new Map<string, DetectedRelationship>()
|
|
534
|
+
|
|
535
|
+
for (const rel of allRelationships) {
|
|
536
|
+
const key = `${rel.relationship.sourceMemoryId}:${rel.relationship.targetMemoryId}`
|
|
537
|
+
const existing = relationshipMap.get(key)
|
|
538
|
+
|
|
539
|
+
if (!existing || rel.relationship.confidence > existing.relationship.confidence) {
|
|
540
|
+
relationshipMap.set(key, rel)
|
|
541
|
+
}
|
|
542
|
+
}
|
|
543
|
+
|
|
544
|
+
return Array.from(relationshipMap.values())
|
|
545
|
+
}
|
|
546
|
+
|
|
547
|
+
// ============================================================================
|
|
548
|
+
// Main Detection API
|
|
549
|
+
// ============================================================================
|
|
550
|
+
|
|
551
|
+
/**
|
|
552
|
+
* Detect relationships for a new memory.
|
|
553
|
+
* This is the main entry point for relationship detection.
|
|
554
|
+
*
|
|
555
|
+
* @param newMemory - The new memory to analyze
|
|
556
|
+
* @param options - Optional filters
|
|
557
|
+
* @returns Detection result with relationships, superseded IDs, and contradictions
|
|
558
|
+
*/
|
|
559
|
+
async detectRelationships(
|
|
560
|
+
newMemory: Memory,
|
|
561
|
+
options: {
|
|
562
|
+
containerTag?: string
|
|
563
|
+
excludeIds?: string[]
|
|
564
|
+
} = {}
|
|
565
|
+
): Promise<RelationshipDetectionResult> {
|
|
566
|
+
const startTime = Date.now()
|
|
567
|
+
const stats: RelationshipDetectionStats = {
|
|
568
|
+
candidatesEvaluated: 0,
|
|
569
|
+
relationshipsDetected: 0,
|
|
570
|
+
byType: {
|
|
571
|
+
updates: 0,
|
|
572
|
+
extends: 0,
|
|
573
|
+
derives: 0,
|
|
574
|
+
contradicts: 0,
|
|
575
|
+
related: 0,
|
|
576
|
+
supersedes: 0,
|
|
577
|
+
},
|
|
578
|
+
llmVerifications: 0,
|
|
579
|
+
processingTimeMs: 0,
|
|
580
|
+
fromCache: false,
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
try {
|
|
584
|
+
logger.debug('Detecting relationships for memory', {
|
|
585
|
+
memoryId: newMemory.id,
|
|
586
|
+
contentPreview: newMemory.content.substring(0, 50),
|
|
587
|
+
})
|
|
588
|
+
|
|
589
|
+
// Step 1: Get embedding for new memory
|
|
590
|
+
const embedding = await this.getOrGenerateEmbedding(newMemory)
|
|
591
|
+
|
|
592
|
+
// Step 2: Find similar memories via vector search
|
|
593
|
+
const minThreshold = Math.min(...Object.values(this.config.thresholds))
|
|
594
|
+
const similarResults = await this.vectorStore.findSimilar(embedding, this.config.maxCandidates, minThreshold, {
|
|
595
|
+
containerTag: options.containerTag,
|
|
596
|
+
excludeIds: [...(options.excludeIds || []), newMemory.id],
|
|
597
|
+
})
|
|
598
|
+
|
|
599
|
+
stats.candidatesEvaluated = similarResults.length
|
|
600
|
+
|
|
601
|
+
if (similarResults.length === 0) {
|
|
602
|
+
logger.debug('No similar memories found', { memoryId: newMemory.id })
|
|
603
|
+
return {
|
|
604
|
+
sourceMemory: newMemory,
|
|
605
|
+
relationships: [],
|
|
606
|
+
supersededMemoryIds: [],
|
|
607
|
+
contradictions: [],
|
|
608
|
+
stats: {
|
|
609
|
+
...stats,
|
|
610
|
+
processingTimeMs: Date.now() - startTime,
|
|
611
|
+
},
|
|
612
|
+
}
|
|
613
|
+
}
|
|
614
|
+
|
|
615
|
+
// Step 3: Build candidates with full scoring
|
|
616
|
+
const candidates = await this.buildCandidates(newMemory, similarResults)
|
|
617
|
+
|
|
618
|
+
// Step 4: Run detection approaches
|
|
619
|
+
const similarityRels = await this.detectBySimilarity(newMemory, candidates)
|
|
620
|
+
const temporalRels = this.config.temporalWeight > 0 ? await this.detectByTemporal(newMemory, candidates) : []
|
|
621
|
+
const entityRels =
|
|
622
|
+
this.config.entityOverlapWeight > 0 ? await this.detectByEntityOverlap(newMemory, candidates) : []
|
|
623
|
+
|
|
624
|
+
// Merge results from all approaches
|
|
625
|
+
const allDetectedRelationships = this.mergeRelationships([...similarityRels, ...temporalRels, ...entityRels])
|
|
626
|
+
|
|
627
|
+
// Step 5: Process results
|
|
628
|
+
const relationships = allDetectedRelationships
|
|
629
|
+
const supersededMemoryIds: string[] = []
|
|
630
|
+
const contradictions: Contradiction[] = []
|
|
631
|
+
|
|
632
|
+
for (const rel of relationships) {
|
|
633
|
+
stats.byType[rel.relationship.type]++
|
|
634
|
+
stats.relationshipsDetected++
|
|
635
|
+
|
|
636
|
+
if (rel.llmVerified) {
|
|
637
|
+
stats.llmVerifications++
|
|
638
|
+
}
|
|
639
|
+
|
|
640
|
+
// Track superseded memories
|
|
641
|
+
if (rel.relationship.type === 'updates' || rel.relationship.type === 'supersedes') {
|
|
642
|
+
supersededMemoryIds.push(rel.relationship.targetMemoryId)
|
|
643
|
+
}
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
// Step 6: Detect contradictions if enabled
|
|
647
|
+
if (this.config.enableContradictionDetection) {
|
|
648
|
+
const detectedContradictions = await this.detectContradictions(
|
|
649
|
+
newMemory,
|
|
650
|
+
candidates.filter((c) => c.vectorSimilarity >= this.config.thresholds.contradicts)
|
|
651
|
+
)
|
|
652
|
+
contradictions.push(...detectedContradictions)
|
|
653
|
+
}
|
|
654
|
+
|
|
655
|
+
stats.processingTimeMs = Date.now() - startTime
|
|
656
|
+
|
|
657
|
+
logger.info('Relationship detection complete', {
|
|
658
|
+
memoryId: newMemory.id,
|
|
659
|
+
stats,
|
|
660
|
+
})
|
|
661
|
+
|
|
662
|
+
return {
|
|
663
|
+
sourceMemory: newMemory,
|
|
664
|
+
relationships,
|
|
665
|
+
supersededMemoryIds: [...new Set(supersededMemoryIds)],
|
|
666
|
+
contradictions,
|
|
667
|
+
stats,
|
|
668
|
+
}
|
|
669
|
+
} catch (error) {
|
|
670
|
+
logger.errorWithException('Relationship detection failed', error, {
|
|
671
|
+
memoryId: newMemory.id,
|
|
672
|
+
})
|
|
673
|
+
throw AppError.from(error, ErrorCode.INTERNAL_ERROR)
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
|
|
677
|
+
/**
|
|
678
|
+
* Batch detect relationships for multiple memories.
|
|
679
|
+
* More efficient than calling detectRelationships for each memory.
|
|
680
|
+
*/
|
|
681
|
+
async batchDetectRelationships(
|
|
682
|
+
memories: Memory[],
|
|
683
|
+
options: {
|
|
684
|
+
containerTag?: string
|
|
685
|
+
} = {}
|
|
686
|
+
): Promise<RelationshipDetectionResult[]> {
|
|
687
|
+
const results: RelationshipDetectionResult[] = []
|
|
688
|
+
const processedIds = new Set<string>()
|
|
689
|
+
|
|
690
|
+
// Process in batches
|
|
691
|
+
for (let i = 0; i < memories.length; i += this.config.batchSize) {
|
|
692
|
+
const batch = memories.slice(i, i + this.config.batchSize)
|
|
693
|
+
|
|
694
|
+
const batchResults = await Promise.all(
|
|
695
|
+
batch.map((memory) =>
|
|
696
|
+
this.detectRelationships(memory, {
|
|
697
|
+
containerTag: options.containerTag,
|
|
698
|
+
excludeIds: [...processedIds],
|
|
699
|
+
})
|
|
700
|
+
)
|
|
701
|
+
)
|
|
702
|
+
|
|
703
|
+
for (const result of batchResults) {
|
|
704
|
+
results.push(result)
|
|
705
|
+
processedIds.add(result.sourceMemory.id)
|
|
706
|
+
}
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
return results
|
|
710
|
+
}
|
|
711
|
+
|
|
712
|
+
// ============================================================================
|
|
713
|
+
// Contradiction Detection
|
|
714
|
+
// ============================================================================
|
|
715
|
+
|
|
716
|
+
/**
|
|
717
|
+
* Detect contradictions between a memory and candidates.
|
|
718
|
+
*/
|
|
719
|
+
async detectContradictions(memory: Memory, candidates: RelationshipCandidate[]): Promise<Contradiction[]> {
|
|
720
|
+
const contradictions: Contradiction[] = []
|
|
721
|
+
|
|
722
|
+
for (const candidate of candidates) {
|
|
723
|
+
// Check for contradiction indicators in content
|
|
724
|
+
const contradictionScore = this.calculateContradictionScore(
|
|
725
|
+
memory.content,
|
|
726
|
+
candidate.memory.content,
|
|
727
|
+
candidate.vectorSimilarity
|
|
728
|
+
)
|
|
729
|
+
|
|
730
|
+
if (contradictionScore.isContradiction) {
|
|
731
|
+
const contradiction: Contradiction = {
|
|
732
|
+
id: generateId(),
|
|
733
|
+
memoryId1: memory.id,
|
|
734
|
+
memoryId2: candidate.memory.id,
|
|
735
|
+
content1: memory.content,
|
|
736
|
+
content2: candidate.memory.content,
|
|
737
|
+
similarity: candidate.vectorSimilarity,
|
|
738
|
+
confidence: contradictionScore.confidence,
|
|
739
|
+
type: contradictionScore.type,
|
|
740
|
+
description: contradictionScore.description,
|
|
741
|
+
suggestedResolution: this.suggestResolution(memory, candidate.memory),
|
|
742
|
+
detectedAt: new Date(),
|
|
743
|
+
resolved: false,
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
// Optionally verify with LLM
|
|
747
|
+
if (this.llmProvider && this.config.enableLLMVerification) {
|
|
748
|
+
try {
|
|
749
|
+
const llmResult = await this.llmProvider.checkContradiction(memory.content, candidate.memory.content)
|
|
750
|
+
|
|
751
|
+
if (llmResult.isContradiction) {
|
|
752
|
+
contradiction.confidence = llmResult.confidence
|
|
753
|
+
if (llmResult.type) {
|
|
754
|
+
contradiction.type = llmResult.type
|
|
755
|
+
}
|
|
756
|
+
contradiction.description = llmResult.description
|
|
757
|
+
} else {
|
|
758
|
+
// LLM says no contradiction, skip
|
|
759
|
+
continue
|
|
760
|
+
}
|
|
761
|
+
} catch (error) {
|
|
762
|
+
logger.warn('LLM contradiction check failed', {
|
|
763
|
+
error: error instanceof Error ? error.message : 'Unknown',
|
|
764
|
+
})
|
|
765
|
+
}
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
contradictions.push(contradiction)
|
|
769
|
+
}
|
|
770
|
+
}
|
|
771
|
+
|
|
772
|
+
return contradictions
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
/**
|
|
776
|
+
* Check multiple memories for contradictions among themselves.
|
|
777
|
+
*/
|
|
778
|
+
async detectContradictionsInGroup(memories: Memory[]): Promise<Contradiction[]> {
|
|
779
|
+
const contradictions: Contradiction[] = []
|
|
780
|
+
|
|
781
|
+
// Get embeddings for all memories
|
|
782
|
+
const embeddings = await Promise.all(memories.map((m) => this.getOrGenerateEmbedding(m)))
|
|
783
|
+
|
|
784
|
+
// Compare each pair
|
|
785
|
+
for (let i = 0; i < memories.length; i++) {
|
|
786
|
+
for (let j = i + 1; j < memories.length; j++) {
|
|
787
|
+
const memory1 = memories[i]!
|
|
788
|
+
const memory2 = memories[j]!
|
|
789
|
+
const embedding1 = embeddings[i]!
|
|
790
|
+
const embedding2 = embeddings[j]!
|
|
791
|
+
|
|
792
|
+
const similarity = cosineSimilarity(embedding1, embedding2)
|
|
793
|
+
|
|
794
|
+
if (similarity >= this.config.thresholds.contradicts) {
|
|
795
|
+
const candidate: RelationshipCandidate = {
|
|
796
|
+
memory: memory2,
|
|
797
|
+
vectorSimilarity: similarity,
|
|
798
|
+
entityOverlap: 0,
|
|
799
|
+
temporalScore: 0,
|
|
800
|
+
combinedScore: similarity,
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
const detectedContradictions = await this.detectContradictions(memory1, [candidate])
|
|
804
|
+
|
|
805
|
+
contradictions.push(...detectedContradictions)
|
|
806
|
+
}
|
|
807
|
+
}
|
|
808
|
+
}
|
|
809
|
+
|
|
810
|
+
return contradictions
|
|
811
|
+
}
|
|
812
|
+
|
|
813
|
+
// ============================================================================
|
|
814
|
+
// Helper Methods
|
|
815
|
+
// ============================================================================
|
|
816
|
+
|
|
817
|
+
/**
|
|
818
|
+
* Get or generate embedding for a memory
|
|
819
|
+
*/
|
|
820
|
+
private async getOrGenerateEmbedding(memory: Memory): Promise<number[]> {
|
|
821
|
+
if (memory.embedding && memory.embedding.length > 0) {
|
|
822
|
+
return memory.embedding
|
|
823
|
+
}
|
|
824
|
+
|
|
825
|
+
return this.embeddingService.generateEmbedding(memory.content)
|
|
826
|
+
}
|
|
827
|
+
|
|
828
|
+
/**
|
|
829
|
+
* Build full candidates with all scores
|
|
830
|
+
*/
|
|
831
|
+
private async buildCandidates(
|
|
832
|
+
newMemory: Memory,
|
|
833
|
+
searchResults: VectorSearchResult[]
|
|
834
|
+
): Promise<RelationshipCandidate[]> {
|
|
835
|
+
const candidates: RelationshipCandidate[] = []
|
|
836
|
+
const now = Date.now()
|
|
837
|
+
|
|
838
|
+
for (const result of searchResults) {
|
|
839
|
+
// Calculate entity overlap
|
|
840
|
+
const entityOverlap = this.calculateEntityOverlap(
|
|
841
|
+
newMemory.metadata?.entities || [],
|
|
842
|
+
result.memory.metadata?.entities || []
|
|
843
|
+
)
|
|
844
|
+
|
|
845
|
+
// Calculate temporal score (recency bias)
|
|
846
|
+
const timeDiff = now - result.memory.createdAt.getTime()
|
|
847
|
+
const oneWeek = 7 * 24 * 60 * 60 * 1000
|
|
848
|
+
const temporalScore = Math.exp(-timeDiff / oneWeek)
|
|
849
|
+
|
|
850
|
+
// Calculate combined score
|
|
851
|
+
const combinedScore = this.calculateCombinedScore(result.similarity, entityOverlap, temporalScore)
|
|
852
|
+
|
|
853
|
+
candidates.push({
|
|
854
|
+
memory: result.memory,
|
|
855
|
+
vectorSimilarity: result.similarity,
|
|
856
|
+
entityOverlap,
|
|
857
|
+
temporalScore,
|
|
858
|
+
combinedScore,
|
|
859
|
+
})
|
|
860
|
+
}
|
|
861
|
+
|
|
862
|
+
// Sort by combined score
|
|
863
|
+
candidates.sort((a, b) => b.combinedScore - a.combinedScore)
|
|
864
|
+
|
|
865
|
+
return candidates
|
|
866
|
+
}
|
|
867
|
+
|
|
868
|
+
/**
|
|
869
|
+
* Calculate entity overlap between two entity lists
|
|
870
|
+
*/
|
|
871
|
+
private calculateEntityOverlap(entities1: unknown[], entities2: unknown[]): number {
|
|
872
|
+
if (!Array.isArray(entities1) || !Array.isArray(entities2)) return 0
|
|
873
|
+
if (entities1.length === 0 || entities2.length === 0) return 0
|
|
874
|
+
|
|
875
|
+
const names1 = new Set(
|
|
876
|
+
entities1
|
|
877
|
+
.filter((e): e is { name: string } => typeof e === 'object' && e !== null && 'name' in e)
|
|
878
|
+
.map((e) => e.name.toLowerCase())
|
|
879
|
+
)
|
|
880
|
+
const names2 = new Set(
|
|
881
|
+
entities2
|
|
882
|
+
.filter((e): e is { name: string } => typeof e === 'object' && e !== null && 'name' in e)
|
|
883
|
+
.map((e) => e.name.toLowerCase())
|
|
884
|
+
)
|
|
885
|
+
|
|
886
|
+
const intersection = [...names1].filter((n) => names2.has(n)).length
|
|
887
|
+
const union = new Set([...names1, ...names2]).size
|
|
888
|
+
|
|
889
|
+
return union > 0 ? intersection / union : 0
|
|
890
|
+
}
|
|
891
|
+
|
|
892
|
+
/**
|
|
893
|
+
* Calculate combined score from multiple signals
|
|
894
|
+
*/
|
|
895
|
+
private calculateCombinedScore(vectorSimilarity: number, entityOverlap: number, temporalScore: number): number {
|
|
896
|
+
const weights = {
|
|
897
|
+
vector: 1 - this.config.temporalWeight - this.config.entityOverlapWeight,
|
|
898
|
+
temporal: this.config.temporalWeight,
|
|
899
|
+
entity: this.config.entityOverlapWeight,
|
|
900
|
+
}
|
|
901
|
+
|
|
902
|
+
return vectorSimilarity * weights.vector + temporalScore * weights.temporal + entityOverlap * weights.entity
|
|
903
|
+
}
|
|
904
|
+
|
|
905
|
+
/**
|
|
906
|
+
* Calculate contradiction score between two pieces of content
|
|
907
|
+
*/
|
|
908
|
+
private calculateContradictionScore(
|
|
909
|
+
content1: string,
|
|
910
|
+
content2: string,
|
|
911
|
+
similarity: number
|
|
912
|
+
): {
|
|
913
|
+
isContradiction: boolean
|
|
914
|
+
type: ContradictionType
|
|
915
|
+
confidence: number
|
|
916
|
+
description: string
|
|
917
|
+
} {
|
|
918
|
+
const lower1 = content1.toLowerCase()
|
|
919
|
+
const lower2 = content2.toLowerCase()
|
|
920
|
+
|
|
921
|
+
// Check for negation patterns
|
|
922
|
+
const negationPatterns = [
|
|
923
|
+
/\bnot\b/,
|
|
924
|
+
/\bno\b/,
|
|
925
|
+
/\bnever\b/,
|
|
926
|
+
/\bwon't\b/,
|
|
927
|
+
/\bdon't\b/,
|
|
928
|
+
/\bdoesn't\b/,
|
|
929
|
+
/\bisn't\b/,
|
|
930
|
+
/\baren't\b/,
|
|
931
|
+
]
|
|
932
|
+
|
|
933
|
+
const hasNegation1 = negationPatterns.some((p) => p.test(lower1))
|
|
934
|
+
const hasNegation2 = negationPatterns.some((p) => p.test(lower2))
|
|
935
|
+
|
|
936
|
+
// XOR negation (one has negation, other doesn't) with high similarity = potential contradiction
|
|
937
|
+
if (hasNegation1 !== hasNegation2 && similarity >= 0.75) {
|
|
938
|
+
return {
|
|
939
|
+
isContradiction: true,
|
|
940
|
+
type: 'factual',
|
|
941
|
+
confidence: similarity * 0.9,
|
|
942
|
+
description: 'Potentially contradictory statements detected (negation asymmetry)',
|
|
943
|
+
}
|
|
944
|
+
}
|
|
945
|
+
|
|
946
|
+
// Check for opposite adjectives/adverbs
|
|
947
|
+
const opposites: [RegExp, RegExp][] = [
|
|
948
|
+
[/\bgood\b/, /\bbad\b/],
|
|
949
|
+
[/\bhigh\b/, /\blow\b/],
|
|
950
|
+
[/\bfast\b/, /\bslow\b/],
|
|
951
|
+
[/\btrue\b/, /\bfalse\b/],
|
|
952
|
+
[/\byes\b/, /\bno\b/],
|
|
953
|
+
[/\blove\b/, /\bhate\b/],
|
|
954
|
+
[/\blike\b/, /\bdislike\b/],
|
|
955
|
+
[/\bprefer\b/, /\bavoid\b/],
|
|
956
|
+
]
|
|
957
|
+
|
|
958
|
+
for (const [pattern1, pattern2] of opposites) {
|
|
959
|
+
if ((pattern1.test(lower1) && pattern2.test(lower2)) || (pattern2.test(lower1) && pattern1.test(lower2))) {
|
|
960
|
+
return {
|
|
961
|
+
isContradiction: true,
|
|
962
|
+
type: 'semantic',
|
|
963
|
+
confidence: similarity * 0.85,
|
|
964
|
+
description: 'Semantically opposite statements detected',
|
|
965
|
+
}
|
|
966
|
+
}
|
|
967
|
+
}
|
|
968
|
+
|
|
969
|
+
// Check for temporal contradiction indicators
|
|
970
|
+
const temporalPatterns = [/\bused to\b/, /\bno longer\b/, /\bpreviously\b/, /\bformerly\b/]
|
|
971
|
+
|
|
972
|
+
if (temporalPatterns.some((p) => p.test(lower1) || p.test(lower2)) && similarity >= 0.7) {
|
|
973
|
+
return {
|
|
974
|
+
isContradiction: true,
|
|
975
|
+
type: 'temporal',
|
|
976
|
+
confidence: similarity * 0.8,
|
|
977
|
+
description: 'Temporal update detected - information may have changed',
|
|
978
|
+
}
|
|
979
|
+
}
|
|
980
|
+
|
|
981
|
+
return {
|
|
982
|
+
isContradiction: false,
|
|
983
|
+
type: 'partial',
|
|
984
|
+
confidence: 0,
|
|
985
|
+
description: 'No contradiction detected',
|
|
986
|
+
}
|
|
987
|
+
}
|
|
988
|
+
|
|
989
|
+
/**
|
|
990
|
+
* Suggest resolution for a contradiction
|
|
991
|
+
*/
|
|
992
|
+
private suggestResolution(memory1: Memory, memory2: Memory): ContradictionResolution {
|
|
993
|
+
// Prefer newer information by default
|
|
994
|
+
const isMemory1Newer = memory1.createdAt > memory2.createdAt
|
|
995
|
+
|
|
996
|
+
// Check confidence levels
|
|
997
|
+
const confidence1 = memory1.confidence ?? 0.5
|
|
998
|
+
const confidence2 = memory2.confidence ?? 0.5
|
|
999
|
+
|
|
1000
|
+
if (Math.abs(confidence1 - confidence2) > 0.2) {
|
|
1001
|
+
// Significant confidence difference - keep higher confidence
|
|
1002
|
+
return {
|
|
1003
|
+
action: confidence1 > confidence2 ? 'keep_newer' : 'keep_older',
|
|
1004
|
+
reason: `Higher confidence memory (${Math.max(confidence1, confidence2).toFixed(2)}) should be preferred`,
|
|
1005
|
+
confidence: 0.7,
|
|
1006
|
+
}
|
|
1007
|
+
}
|
|
1008
|
+
|
|
1009
|
+
if (isMemory1Newer) {
|
|
1010
|
+
return {
|
|
1011
|
+
action: 'keep_newer',
|
|
1012
|
+
reason: 'Newer information typically supersedes older information',
|
|
1013
|
+
confidence: 0.6,
|
|
1014
|
+
}
|
|
1015
|
+
}
|
|
1016
|
+
|
|
1017
|
+
return {
|
|
1018
|
+
action: 'manual_review',
|
|
1019
|
+
reason: 'Unable to automatically determine which memory is more accurate',
|
|
1020
|
+
confidence: 0.4,
|
|
1021
|
+
}
|
|
1022
|
+
}
|
|
1023
|
+
|
|
1024
|
+
// ============================================================================
|
|
1025
|
+
// Cache Management
|
|
1026
|
+
// ============================================================================
|
|
1027
|
+
|
|
1028
|
+
/**
|
|
1029
|
+
* Get cached relationship score
|
|
1030
|
+
*/
|
|
1031
|
+
getCachedScore(sourceId: string, targetId: string): CachedRelationshipScore | null {
|
|
1032
|
+
const key = generateCacheKey(sourceId, targetId)
|
|
1033
|
+
const cached = this.cache.get(key)
|
|
1034
|
+
|
|
1035
|
+
if (cached && Date.now() - cached.cachedAt < this.config.cacheTTL) {
|
|
1036
|
+
return cached
|
|
1037
|
+
}
|
|
1038
|
+
|
|
1039
|
+
// Cache expired
|
|
1040
|
+
if (cached) {
|
|
1041
|
+
this.cache.delete(key)
|
|
1042
|
+
}
|
|
1043
|
+
|
|
1044
|
+
return null
|
|
1045
|
+
}
|
|
1046
|
+
|
|
1047
|
+
/**
|
|
1048
|
+
* Cache a relationship score
|
|
1049
|
+
*/
|
|
1050
|
+
cacheScore(sourceId: string, targetId: string, score: number, type: RelationshipType | null): void {
|
|
1051
|
+
const key = generateCacheKey(sourceId, targetId)
|
|
1052
|
+
this.cache.set(key, {
|
|
1053
|
+
sourceId,
|
|
1054
|
+
targetId,
|
|
1055
|
+
score,
|
|
1056
|
+
type,
|
|
1057
|
+
cachedAt: Date.now(),
|
|
1058
|
+
})
|
|
1059
|
+
}
|
|
1060
|
+
|
|
1061
|
+
/**
|
|
1062
|
+
* Clear the cache
|
|
1063
|
+
*/
|
|
1064
|
+
clearCache(): void {
|
|
1065
|
+
this.cache.clear()
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
/**
|
|
1069
|
+
* Get cache statistics
|
|
1070
|
+
*/
|
|
1071
|
+
getCacheStats(): { size: number; oldestEntry: number | null } {
|
|
1072
|
+
let oldest: number | null = null
|
|
1073
|
+
|
|
1074
|
+
for (const entry of this.cache.values()) {
|
|
1075
|
+
if (oldest === null || entry.cachedAt < oldest) {
|
|
1076
|
+
oldest = entry.cachedAt
|
|
1077
|
+
}
|
|
1078
|
+
}
|
|
1079
|
+
|
|
1080
|
+
return {
|
|
1081
|
+
size: this.cache.size,
|
|
1082
|
+
oldestEntry: oldest,
|
|
1083
|
+
}
|
|
1084
|
+
}
|
|
1085
|
+
|
|
1086
|
+
// ============================================================================
|
|
1087
|
+
// Configuration
|
|
1088
|
+
// ============================================================================
|
|
1089
|
+
|
|
1090
|
+
/**
|
|
1091
|
+
* Get current configuration
|
|
1092
|
+
*/
|
|
1093
|
+
getConfig(): RelationshipConfig {
|
|
1094
|
+
return { ...this.config }
|
|
1095
|
+
}
|
|
1096
|
+
|
|
1097
|
+
/**
|
|
1098
|
+
* Update configuration
|
|
1099
|
+
*/
|
|
1100
|
+
updateConfig(updates: Partial<RelationshipConfig>): void {
|
|
1101
|
+
Object.assign(this.config, updates)
|
|
1102
|
+
logger.debug('Configuration updated', { updates })
|
|
1103
|
+
}
|
|
1104
|
+
|
|
1105
|
+
/**
|
|
1106
|
+
* Set LLM provider
|
|
1107
|
+
*/
|
|
1108
|
+
setLLMProvider(provider: LLMProvider): void {
|
|
1109
|
+
this.llmProvider = provider
|
|
1110
|
+
}
|
|
1111
|
+
}
|
|
1112
|
+
|
|
1113
|
+
// ============================================================================
|
|
1114
|
+
// Factory Functions
|
|
1115
|
+
// ============================================================================
|
|
1116
|
+
|
|
1117
|
+
/**
|
|
1118
|
+
* Create an embedding relationship detector with default configuration
|
|
1119
|
+
*/
|
|
1120
|
+
export function createEmbeddingRelationshipDetector(
|
|
1121
|
+
embeddingService: EmbeddingService,
|
|
1122
|
+
vectorStore?: VectorStore,
|
|
1123
|
+
config?: Partial<RelationshipConfig>,
|
|
1124
|
+
llmProvider?: LLMProvider
|
|
1125
|
+
): EmbeddingRelationshipDetector {
|
|
1126
|
+
const store = vectorStore ?? new InMemoryVectorStoreAdapter()
|
|
1127
|
+
return new EmbeddingRelationshipDetector(embeddingService, store, config, llmProvider)
|
|
1128
|
+
}
|