@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,268 @@
1
+ /**
2
+ * Relationship Detection Module
3
+ *
4
+ * Embedding-based relationship detection for memories.
5
+ * Provides vector similarity, temporal analysis, entity overlap,
6
+ * and optional LLM verification for relationship detection.
7
+ *
8
+ * @example
9
+ * ```typescript
10
+ * import {
11
+ * createEmbeddingRelationshipDetector,
12
+ * EmbeddingRelationshipDetector,
13
+ * DEFAULT_RELATIONSHIP_CONFIG,
14
+ * } from './relationships';
15
+ *
16
+ * const detector = createEmbeddingRelationshipDetector(embeddingService);
17
+ * const result = await detector.detectRelationships(newMemory);
18
+ *
19
+ * console.log(`Found ${result.relationships.length} relationships`);
20
+ * console.log(`Superseded memories: ${result.supersededMemoryIds.join(', ')}`);
21
+ * console.log(`Contradictions: ${result.contradictions.length}`);
22
+ * ```
23
+ */
24
+
25
+ // ============================================================================
26
+ // Types
27
+ // ============================================================================
28
+
29
+ export type {
30
+ // Configuration
31
+ RelationshipConfig,
32
+ RelationshipThresholds,
33
+
34
+ // Detection Results
35
+ RelationshipCandidate,
36
+ DetectedRelationship,
37
+ RelationshipDetectionResult,
38
+ RelationshipDetectionStats,
39
+
40
+ // Contradictions
41
+ Contradiction,
42
+ ContradictionType,
43
+ ContradictionResolution,
44
+
45
+ // Strategy Type (kept for metadata only)
46
+ DetectionStrategyType,
47
+
48
+ // Vector Store
49
+ VectorStore,
50
+ VectorSearchResult,
51
+
52
+ // LLM Provider
53
+ LLMProvider,
54
+ LLMVerificationRequest,
55
+ LLMVerificationResponse,
56
+
57
+ // Cache
58
+ CachedRelationshipScore,
59
+ } from './types.js'
60
+
61
+ export { DEFAULT_RELATIONSHIP_CONFIG, DEFAULT_RELATIONSHIP_THRESHOLDS, generateCacheKey } from './types.js'
62
+
63
+ // ============================================================================
64
+ // Helper Functions
65
+ // ============================================================================
66
+
67
+ export {
68
+ createDetectedRelationship,
69
+ hasUpdateIndicators,
70
+ hasExtensionIndicators,
71
+ hasContradictionIndicators,
72
+ hasSupersessionIndicators,
73
+ hasCausalIndicators,
74
+ } from './strategies.js'
75
+
76
+ // ============================================================================
77
+ // Detector
78
+ // ============================================================================
79
+
80
+ export {
81
+ EmbeddingRelationshipDetector,
82
+ InMemoryVectorStoreAdapter,
83
+ createEmbeddingRelationshipDetector,
84
+ } from './detector.js'
85
+
86
+ // ============================================================================
87
+ // Factory Functions
88
+ // ============================================================================
89
+
90
+ import type { EmbeddingService } from '../embedding.service.js'
91
+ import type { Memory } from '../memory.types.js'
92
+ import { EmbeddingRelationshipDetector, InMemoryVectorStoreAdapter } from './detector.js'
93
+ import type { RelationshipConfig, VectorStore, LLMProvider, RelationshipDetectionResult } from './types.js'
94
+
95
+ /**
96
+ * Singleton instance (lazy initialization)
97
+ */
98
+ let _detectorInstance: EmbeddingRelationshipDetector | null = null
99
+ let _vectorStoreInstance: InMemoryVectorStoreAdapter | null = null
100
+
101
+ /**
102
+ * Get or create the shared vector store instance
103
+ */
104
+ export function getSharedVectorStore(): InMemoryVectorStoreAdapter {
105
+ if (!_vectorStoreInstance) {
106
+ _vectorStoreInstance = new InMemoryVectorStoreAdapter()
107
+ }
108
+ return _vectorStoreInstance
109
+ }
110
+
111
+ /**
112
+ * Get the singleton relationship detector instance
113
+ *
114
+ * @param embeddingService - Required on first call to initialize
115
+ * @param config - Optional configuration overrides
116
+ */
117
+ export function getRelationshipDetector(
118
+ embeddingService?: EmbeddingService,
119
+ config?: Partial<RelationshipConfig>
120
+ ): EmbeddingRelationshipDetector {
121
+ if (!_detectorInstance) {
122
+ if (!embeddingService) {
123
+ throw new Error('EmbeddingService is required for first initialization of RelationshipDetector')
124
+ }
125
+ _detectorInstance = new EmbeddingRelationshipDetector(embeddingService, getSharedVectorStore(), config)
126
+ }
127
+ return _detectorInstance
128
+ }
129
+
130
+ /**
131
+ * Reset singleton instances (for testing)
132
+ */
133
+ export function resetRelationshipDetector(): void {
134
+ _detectorInstance = null
135
+ _vectorStoreInstance = null
136
+ }
137
+
138
+ /**
139
+ * Create a standalone relationship detector
140
+ * (not using singleton pattern)
141
+ */
142
+ export function createRelationshipDetector(
143
+ embeddingService: EmbeddingService,
144
+ options?: {
145
+ vectorStore?: VectorStore
146
+ config?: Partial<RelationshipConfig>
147
+ llmProvider?: LLMProvider
148
+ }
149
+ ): EmbeddingRelationshipDetector {
150
+ const vectorStore = options?.vectorStore ?? new InMemoryVectorStoreAdapter()
151
+ return new EmbeddingRelationshipDetector(embeddingService, vectorStore, options?.config, options?.llmProvider)
152
+ }
153
+
154
+ // ============================================================================
155
+ // Convenience Functions
156
+ // ============================================================================
157
+
158
+ /**
159
+ * Quick relationship detection for a single memory.
160
+ * Uses the singleton detector instance.
161
+ *
162
+ * @param memory - Memory to analyze
163
+ * @param embeddingService - Required if detector not yet initialized
164
+ * @param containerTag - Optional container tag filter
165
+ */
166
+ export async function detectRelationshipsQuick(
167
+ memory: Memory,
168
+ embeddingService?: EmbeddingService,
169
+ containerTag?: string
170
+ ): Promise<RelationshipDetectionResult> {
171
+ const detector = getRelationshipDetector(embeddingService)
172
+ return detector.detectRelationships(memory, { containerTag })
173
+ }
174
+
175
+ /**
176
+ * Quick batch relationship detection.
177
+ * Uses the singleton detector instance.
178
+ *
179
+ * @param memories - Memories to analyze
180
+ * @param embeddingService - Required if detector not yet initialized
181
+ * @param containerTag - Optional container tag filter
182
+ */
183
+ export async function batchDetectRelationshipsQuick(
184
+ memories: Memory[],
185
+ embeddingService?: EmbeddingService,
186
+ containerTag?: string
187
+ ): Promise<RelationshipDetectionResult[]> {
188
+ const detector = getRelationshipDetector(embeddingService)
189
+ return detector.batchDetectRelationships(memories, { containerTag })
190
+ }
191
+
192
+ /**
193
+ * Quick contradiction detection among a group of memories.
194
+ * Uses the singleton detector instance.
195
+ *
196
+ * @param memories - Memories to check for contradictions
197
+ * @param embeddingService - Required if detector not yet initialized
198
+ */
199
+ export async function detectContradictionsQuick(
200
+ memories: Memory[],
201
+ embeddingService?: EmbeddingService
202
+ ): Promise<import('./types.js').Contradiction[]> {
203
+ const detector = getRelationshipDetector(embeddingService)
204
+ return detector.detectContradictionsInGroup(memories)
205
+ }
206
+
207
+ // ============================================================================
208
+ // Integration Helpers
209
+ // ============================================================================
210
+
211
+ /**
212
+ * Index a memory in the shared vector store.
213
+ * Call this when adding new memories to enable relationship detection.
214
+ *
215
+ * @param memory - Memory to index
216
+ * @param embedding - Pre-computed embedding (optional)
217
+ * @param embeddingService - Service to generate embedding if not provided
218
+ */
219
+ export async function indexMemoryForRelationships(
220
+ memory: Memory,
221
+ embedding?: number[],
222
+ embeddingService?: EmbeddingService
223
+ ): Promise<void> {
224
+ const store = getSharedVectorStore()
225
+
226
+ if (embedding) {
227
+ store.addMemory(memory, embedding)
228
+ } else if (embeddingService) {
229
+ const generatedEmbedding = await embeddingService.generateEmbedding(memory.content)
230
+ store.addMemory(memory, generatedEmbedding)
231
+ } else if (memory.embedding && memory.embedding.length > 0) {
232
+ store.addMemory(memory, memory.embedding)
233
+ } else {
234
+ throw new Error('Either embedding or embeddingService must be provided if memory has no embedding')
235
+ }
236
+ }
237
+
238
+ /**
239
+ * Remove a memory from the shared vector store.
240
+ *
241
+ * @param memoryId - ID of memory to remove
242
+ */
243
+ export function removeMemoryFromRelationshipIndex(memoryId: string): boolean {
244
+ const store = getSharedVectorStore()
245
+ return store.removeMemory(memoryId)
246
+ }
247
+
248
+ /**
249
+ * Clear all memories from the shared vector store.
250
+ */
251
+ export function clearRelationshipIndex(): void {
252
+ const store = getSharedVectorStore()
253
+ store.clear()
254
+ }
255
+
256
+ // ============================================================================
257
+ // Memory Service Integration
258
+ // ============================================================================
259
+
260
+ export {
261
+ EnhancedMemoryService,
262
+ createEnhancedMemoryService,
263
+ getEnhancedMemoryService,
264
+ resetEnhancedMemoryService,
265
+ enhancedMemoryService,
266
+ type EnhancedMemoryServiceConfig,
267
+ DEFAULT_ENHANCED_CONFIG,
268
+ } from './memory-integration.js'