@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,268 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Relationship Detection Module
|
|
3
|
+
*
|
|
4
|
+
* Embedding-based relationship detection for memories.
|
|
5
|
+
* Provides vector similarity, temporal analysis, entity overlap,
|
|
6
|
+
* and optional LLM verification for relationship detection.
|
|
7
|
+
*
|
|
8
|
+
* @example
|
|
9
|
+
* ```typescript
|
|
10
|
+
* import {
|
|
11
|
+
* createEmbeddingRelationshipDetector,
|
|
12
|
+
* EmbeddingRelationshipDetector,
|
|
13
|
+
* DEFAULT_RELATIONSHIP_CONFIG,
|
|
14
|
+
* } from './relationships';
|
|
15
|
+
*
|
|
16
|
+
* const detector = createEmbeddingRelationshipDetector(embeddingService);
|
|
17
|
+
* const result = await detector.detectRelationships(newMemory);
|
|
18
|
+
*
|
|
19
|
+
* console.log(`Found ${result.relationships.length} relationships`);
|
|
20
|
+
* console.log(`Superseded memories: ${result.supersededMemoryIds.join(', ')}`);
|
|
21
|
+
* console.log(`Contradictions: ${result.contradictions.length}`);
|
|
22
|
+
* ```
|
|
23
|
+
*/
|
|
24
|
+
|
|
25
|
+
// ============================================================================
|
|
26
|
+
// Types
|
|
27
|
+
// ============================================================================
|
|
28
|
+
|
|
29
|
+
export type {
|
|
30
|
+
// Configuration
|
|
31
|
+
RelationshipConfig,
|
|
32
|
+
RelationshipThresholds,
|
|
33
|
+
|
|
34
|
+
// Detection Results
|
|
35
|
+
RelationshipCandidate,
|
|
36
|
+
DetectedRelationship,
|
|
37
|
+
RelationshipDetectionResult,
|
|
38
|
+
RelationshipDetectionStats,
|
|
39
|
+
|
|
40
|
+
// Contradictions
|
|
41
|
+
Contradiction,
|
|
42
|
+
ContradictionType,
|
|
43
|
+
ContradictionResolution,
|
|
44
|
+
|
|
45
|
+
// Strategy Type (kept for metadata only)
|
|
46
|
+
DetectionStrategyType,
|
|
47
|
+
|
|
48
|
+
// Vector Store
|
|
49
|
+
VectorStore,
|
|
50
|
+
VectorSearchResult,
|
|
51
|
+
|
|
52
|
+
// LLM Provider
|
|
53
|
+
LLMProvider,
|
|
54
|
+
LLMVerificationRequest,
|
|
55
|
+
LLMVerificationResponse,
|
|
56
|
+
|
|
57
|
+
// Cache
|
|
58
|
+
CachedRelationshipScore,
|
|
59
|
+
} from './types.js'
|
|
60
|
+
|
|
61
|
+
export { DEFAULT_RELATIONSHIP_CONFIG, DEFAULT_RELATIONSHIP_THRESHOLDS, generateCacheKey } from './types.js'
|
|
62
|
+
|
|
63
|
+
// ============================================================================
|
|
64
|
+
// Helper Functions
|
|
65
|
+
// ============================================================================
|
|
66
|
+
|
|
67
|
+
export {
|
|
68
|
+
createDetectedRelationship,
|
|
69
|
+
hasUpdateIndicators,
|
|
70
|
+
hasExtensionIndicators,
|
|
71
|
+
hasContradictionIndicators,
|
|
72
|
+
hasSupersessionIndicators,
|
|
73
|
+
hasCausalIndicators,
|
|
74
|
+
} from './strategies.js'
|
|
75
|
+
|
|
76
|
+
// ============================================================================
|
|
77
|
+
// Detector
|
|
78
|
+
// ============================================================================
|
|
79
|
+
|
|
80
|
+
export {
|
|
81
|
+
EmbeddingRelationshipDetector,
|
|
82
|
+
InMemoryVectorStoreAdapter,
|
|
83
|
+
createEmbeddingRelationshipDetector,
|
|
84
|
+
} from './detector.js'
|
|
85
|
+
|
|
86
|
+
// ============================================================================
|
|
87
|
+
// Factory Functions
|
|
88
|
+
// ============================================================================
|
|
89
|
+
|
|
90
|
+
import type { EmbeddingService } from '../embedding.service.js'
|
|
91
|
+
import type { Memory } from '../memory.types.js'
|
|
92
|
+
import { EmbeddingRelationshipDetector, InMemoryVectorStoreAdapter } from './detector.js'
|
|
93
|
+
import type { RelationshipConfig, VectorStore, LLMProvider, RelationshipDetectionResult } from './types.js'
|
|
94
|
+
|
|
95
|
+
/**
|
|
96
|
+
* Singleton instance (lazy initialization)
|
|
97
|
+
*/
|
|
98
|
+
let _detectorInstance: EmbeddingRelationshipDetector | null = null
|
|
99
|
+
let _vectorStoreInstance: InMemoryVectorStoreAdapter | null = null
|
|
100
|
+
|
|
101
|
+
/**
|
|
102
|
+
* Get or create the shared vector store instance
|
|
103
|
+
*/
|
|
104
|
+
export function getSharedVectorStore(): InMemoryVectorStoreAdapter {
|
|
105
|
+
if (!_vectorStoreInstance) {
|
|
106
|
+
_vectorStoreInstance = new InMemoryVectorStoreAdapter()
|
|
107
|
+
}
|
|
108
|
+
return _vectorStoreInstance
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* Get the singleton relationship detector instance
|
|
113
|
+
*
|
|
114
|
+
* @param embeddingService - Required on first call to initialize
|
|
115
|
+
* @param config - Optional configuration overrides
|
|
116
|
+
*/
|
|
117
|
+
export function getRelationshipDetector(
|
|
118
|
+
embeddingService?: EmbeddingService,
|
|
119
|
+
config?: Partial<RelationshipConfig>
|
|
120
|
+
): EmbeddingRelationshipDetector {
|
|
121
|
+
if (!_detectorInstance) {
|
|
122
|
+
if (!embeddingService) {
|
|
123
|
+
throw new Error('EmbeddingService is required for first initialization of RelationshipDetector')
|
|
124
|
+
}
|
|
125
|
+
_detectorInstance = new EmbeddingRelationshipDetector(embeddingService, getSharedVectorStore(), config)
|
|
126
|
+
}
|
|
127
|
+
return _detectorInstance
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Reset singleton instances (for testing)
|
|
132
|
+
*/
|
|
133
|
+
export function resetRelationshipDetector(): void {
|
|
134
|
+
_detectorInstance = null
|
|
135
|
+
_vectorStoreInstance = null
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/**
|
|
139
|
+
* Create a standalone relationship detector
|
|
140
|
+
* (not using singleton pattern)
|
|
141
|
+
*/
|
|
142
|
+
export function createRelationshipDetector(
|
|
143
|
+
embeddingService: EmbeddingService,
|
|
144
|
+
options?: {
|
|
145
|
+
vectorStore?: VectorStore
|
|
146
|
+
config?: Partial<RelationshipConfig>
|
|
147
|
+
llmProvider?: LLMProvider
|
|
148
|
+
}
|
|
149
|
+
): EmbeddingRelationshipDetector {
|
|
150
|
+
const vectorStore = options?.vectorStore ?? new InMemoryVectorStoreAdapter()
|
|
151
|
+
return new EmbeddingRelationshipDetector(embeddingService, vectorStore, options?.config, options?.llmProvider)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// ============================================================================
|
|
155
|
+
// Convenience Functions
|
|
156
|
+
// ============================================================================
|
|
157
|
+
|
|
158
|
+
/**
|
|
159
|
+
* Quick relationship detection for a single memory.
|
|
160
|
+
* Uses the singleton detector instance.
|
|
161
|
+
*
|
|
162
|
+
* @param memory - Memory to analyze
|
|
163
|
+
* @param embeddingService - Required if detector not yet initialized
|
|
164
|
+
* @param containerTag - Optional container tag filter
|
|
165
|
+
*/
|
|
166
|
+
export async function detectRelationshipsQuick(
|
|
167
|
+
memory: Memory,
|
|
168
|
+
embeddingService?: EmbeddingService,
|
|
169
|
+
containerTag?: string
|
|
170
|
+
): Promise<RelationshipDetectionResult> {
|
|
171
|
+
const detector = getRelationshipDetector(embeddingService)
|
|
172
|
+
return detector.detectRelationships(memory, { containerTag })
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Quick batch relationship detection.
|
|
177
|
+
* Uses the singleton detector instance.
|
|
178
|
+
*
|
|
179
|
+
* @param memories - Memories to analyze
|
|
180
|
+
* @param embeddingService - Required if detector not yet initialized
|
|
181
|
+
* @param containerTag - Optional container tag filter
|
|
182
|
+
*/
|
|
183
|
+
export async function batchDetectRelationshipsQuick(
|
|
184
|
+
memories: Memory[],
|
|
185
|
+
embeddingService?: EmbeddingService,
|
|
186
|
+
containerTag?: string
|
|
187
|
+
): Promise<RelationshipDetectionResult[]> {
|
|
188
|
+
const detector = getRelationshipDetector(embeddingService)
|
|
189
|
+
return detector.batchDetectRelationships(memories, { containerTag })
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
/**
|
|
193
|
+
* Quick contradiction detection among a group of memories.
|
|
194
|
+
* Uses the singleton detector instance.
|
|
195
|
+
*
|
|
196
|
+
* @param memories - Memories to check for contradictions
|
|
197
|
+
* @param embeddingService - Required if detector not yet initialized
|
|
198
|
+
*/
|
|
199
|
+
export async function detectContradictionsQuick(
|
|
200
|
+
memories: Memory[],
|
|
201
|
+
embeddingService?: EmbeddingService
|
|
202
|
+
): Promise<import('./types.js').Contradiction[]> {
|
|
203
|
+
const detector = getRelationshipDetector(embeddingService)
|
|
204
|
+
return detector.detectContradictionsInGroup(memories)
|
|
205
|
+
}
|
|
206
|
+
|
|
207
|
+
// ============================================================================
|
|
208
|
+
// Integration Helpers
|
|
209
|
+
// ============================================================================
|
|
210
|
+
|
|
211
|
+
/**
|
|
212
|
+
* Index a memory in the shared vector store.
|
|
213
|
+
* Call this when adding new memories to enable relationship detection.
|
|
214
|
+
*
|
|
215
|
+
* @param memory - Memory to index
|
|
216
|
+
* @param embedding - Pre-computed embedding (optional)
|
|
217
|
+
* @param embeddingService - Service to generate embedding if not provided
|
|
218
|
+
*/
|
|
219
|
+
export async function indexMemoryForRelationships(
|
|
220
|
+
memory: Memory,
|
|
221
|
+
embedding?: number[],
|
|
222
|
+
embeddingService?: EmbeddingService
|
|
223
|
+
): Promise<void> {
|
|
224
|
+
const store = getSharedVectorStore()
|
|
225
|
+
|
|
226
|
+
if (embedding) {
|
|
227
|
+
store.addMemory(memory, embedding)
|
|
228
|
+
} else if (embeddingService) {
|
|
229
|
+
const generatedEmbedding = await embeddingService.generateEmbedding(memory.content)
|
|
230
|
+
store.addMemory(memory, generatedEmbedding)
|
|
231
|
+
} else if (memory.embedding && memory.embedding.length > 0) {
|
|
232
|
+
store.addMemory(memory, memory.embedding)
|
|
233
|
+
} else {
|
|
234
|
+
throw new Error('Either embedding or embeddingService must be provided if memory has no embedding')
|
|
235
|
+
}
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
/**
|
|
239
|
+
* Remove a memory from the shared vector store.
|
|
240
|
+
*
|
|
241
|
+
* @param memoryId - ID of memory to remove
|
|
242
|
+
*/
|
|
243
|
+
export function removeMemoryFromRelationshipIndex(memoryId: string): boolean {
|
|
244
|
+
const store = getSharedVectorStore()
|
|
245
|
+
return store.removeMemory(memoryId)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Clear all memories from the shared vector store.
|
|
250
|
+
*/
|
|
251
|
+
export function clearRelationshipIndex(): void {
|
|
252
|
+
const store = getSharedVectorStore()
|
|
253
|
+
store.clear()
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
// ============================================================================
|
|
257
|
+
// Memory Service Integration
|
|
258
|
+
// ============================================================================
|
|
259
|
+
|
|
260
|
+
export {
|
|
261
|
+
EnhancedMemoryService,
|
|
262
|
+
createEnhancedMemoryService,
|
|
263
|
+
getEnhancedMemoryService,
|
|
264
|
+
resetEnhancedMemoryService,
|
|
265
|
+
enhancedMemoryService,
|
|
266
|
+
type EnhancedMemoryServiceConfig,
|
|
267
|
+
DEFAULT_ENHANCED_CONFIG,
|
|
268
|
+
} from './memory-integration.js'
|