@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,370 @@
1
+ /**
2
+ * Relationship Detection Types
3
+ *
4
+ * Type definitions for embedding-based relationship detection between memories.
5
+ * Supports vector similarity, temporal analysis, entity overlap, and LLM verification.
6
+ */
7
+
8
+ import type { RelationshipType } from '../../types/index.js'
9
+ import type { Memory, Relationship } from '../memory.types.js'
10
+
11
+ // ============================================================================
12
+ // Configuration Types
13
+ // ============================================================================
14
+
15
+ /**
16
+ * Similarity thresholds for different relationship types
17
+ */
18
+ export interface RelationshipThresholds {
19
+ /** Threshold for 'updates' relationship (high similarity, same topic) */
20
+ updates: number
21
+ /** Threshold for 'extends' relationship (moderate similarity) */
22
+ extends: number
23
+ /** Threshold for 'contradicts' relationship (high similarity but opposing) */
24
+ contradicts: number
25
+ /** Threshold for 'supersedes' relationship (very high similarity) */
26
+ supersedes: number
27
+ /** Threshold for 'related' relationship (lower similarity) */
28
+ related: number
29
+ /** Threshold for 'derives' relationship (moderate, causal) */
30
+ derives: number
31
+ }
32
+
33
+ /**
34
+ * Default similarity thresholds
35
+ */
36
+ export const DEFAULT_RELATIONSHIP_THRESHOLDS: RelationshipThresholds = {
37
+ updates: 0.85,
38
+ extends: 0.7,
39
+ contradicts: 0.8,
40
+ supersedes: 0.9,
41
+ related: 0.6,
42
+ derives: 0.65,
43
+ }
44
+
45
+ /**
46
+ * Configuration for the relationship detector
47
+ */
48
+ export interface RelationshipConfig {
49
+ /** Similarity thresholds per relationship type */
50
+ thresholds: RelationshipThresholds
51
+
52
+ /** Maximum candidates to retrieve for comparison */
53
+ maxCandidates: number
54
+
55
+ /** Whether to use LLM for verification of high-confidence relationships */
56
+ enableLLMVerification: boolean
57
+
58
+ /** Minimum confidence to trigger LLM verification */
59
+ llmVerificationThreshold: number
60
+
61
+ /** Temporal weight for recency bias (0-1) */
62
+ temporalWeight: number
63
+
64
+ /** Entity overlap weight for scoring (0-1) */
65
+ entityOverlapWeight: number
66
+
67
+ /** Whether to enable contradiction detection */
68
+ enableContradictionDetection: boolean
69
+
70
+ /** Batch size for processing relationships */
71
+ batchSize: number
72
+
73
+ /** Cache TTL for relationship scores (ms) */
74
+ cacheTTL: number
75
+
76
+ /** Whether to detect causal/derivation relationships */
77
+ enableCausalDetection: boolean
78
+ }
79
+
80
+ /**
81
+ * Default relationship detector configuration
82
+ */
83
+ export const DEFAULT_RELATIONSHIP_CONFIG: RelationshipConfig = {
84
+ thresholds: DEFAULT_RELATIONSHIP_THRESHOLDS,
85
+ maxCandidates: 50,
86
+ enableLLMVerification: false,
87
+ llmVerificationThreshold: 0.85,
88
+ temporalWeight: 0.1,
89
+ entityOverlapWeight: 0.2,
90
+ enableContradictionDetection: true,
91
+ batchSize: 10,
92
+ cacheTTL: 5 * 60 * 1000, // 5 minutes
93
+ enableCausalDetection: true,
94
+ }
95
+
96
+ // ============================================================================
97
+ // Detection Result Types
98
+ // ============================================================================
99
+
100
+ /**
101
+ * Candidate memory for relationship detection
102
+ */
103
+ export interface RelationshipCandidate {
104
+ /** The candidate memory */
105
+ memory: Memory
106
+ /** Vector similarity score (0-1) */
107
+ vectorSimilarity: number
108
+ /** Entity overlap score (0-1) */
109
+ entityOverlap: number
110
+ /** Temporal proximity score (0-1, higher = more recent) */
111
+ temporalScore: number
112
+ /** Combined score */
113
+ combinedScore: number
114
+ }
115
+
116
+ /**
117
+ * Detected relationship with scoring details
118
+ */
119
+ export interface DetectedRelationship {
120
+ /** The relationship object */
121
+ relationship: Relationship
122
+ /** Combined score for the relationship (0-1) */
123
+ score: number
124
+ /** Vector similarity that triggered the detection */
125
+ vectorSimilarity: number
126
+ /** Entity overlap score */
127
+ entityOverlap: number
128
+ /** Temporal proximity score */
129
+ temporalScore: number
130
+ /** Whether LLM verification was applied */
131
+ llmVerified: boolean
132
+ /** LLM verification confidence (if applied) */
133
+ llmConfidence?: number
134
+ /** Strategy that detected this relationship */
135
+ detectionStrategy: DetectionStrategyType
136
+ }
137
+
138
+ /**
139
+ * Result of relationship detection for a memory
140
+ */
141
+ export interface RelationshipDetectionResult {
142
+ /** The source memory analyzed */
143
+ sourceMemory: Memory
144
+ /** All detected relationships */
145
+ relationships: DetectedRelationship[]
146
+ /** Memory IDs that should be marked as superseded */
147
+ supersededMemoryIds: string[]
148
+ /** Detected contradictions (if enabled) */
149
+ contradictions: Contradiction[]
150
+ /** Processing statistics */
151
+ stats: RelationshipDetectionStats
152
+ }
153
+
154
+ /**
155
+ * Statistics for relationship detection
156
+ */
157
+ export interface RelationshipDetectionStats {
158
+ /** Total candidates evaluated */
159
+ candidatesEvaluated: number
160
+ /** Total relationships detected */
161
+ relationshipsDetected: number
162
+ /** Breakdown by relationship type */
163
+ byType: Record<RelationshipType, number>
164
+ /** Number of LLM verifications performed */
165
+ llmVerifications: number
166
+ /** Processing time in milliseconds */
167
+ processingTimeMs: number
168
+ /** Whether results were cached */
169
+ fromCache: boolean
170
+ }
171
+
172
+ // ============================================================================
173
+ // Contradiction Types
174
+ // ============================================================================
175
+
176
+ /**
177
+ * Detected contradiction between memories
178
+ */
179
+ export interface Contradiction {
180
+ /** Unique identifier */
181
+ id: string
182
+ /** First memory ID */
183
+ memoryId1: string
184
+ /** Second memory ID */
185
+ memoryId2: string
186
+ /** Content of first memory */
187
+ content1: string
188
+ /** Content of second memory */
189
+ content2: string
190
+ /** Similarity score between the memories */
191
+ similarity: number
192
+ /** Confidence that these are contradictory (0-1) */
193
+ confidence: number
194
+ /** Type of contradiction */
195
+ type: ContradictionType
196
+ /** Human-readable description of the contradiction */
197
+ description: string
198
+ /** Suggested resolution */
199
+ suggestedResolution?: ContradictionResolution
200
+ /** When the contradiction was detected */
201
+ detectedAt: Date
202
+ /** Whether the contradiction has been resolved */
203
+ resolved: boolean
204
+ /** Resolution notes if resolved */
205
+ resolutionNotes?: string
206
+ }
207
+
208
+ /**
209
+ * Types of contradictions
210
+ */
211
+ export type ContradictionType =
212
+ | 'factual' // Direct factual disagreement
213
+ | 'temporal' // Time-based contradiction (old vs new info)
214
+ | 'preference' // Conflicting preferences
215
+ | 'partial' // Partial overlap with conflicting details
216
+ | 'semantic' // Semantically opposite meanings
217
+
218
+ /**
219
+ * Suggested resolution for a contradiction
220
+ */
221
+ export interface ContradictionResolution {
222
+ /** Recommended action */
223
+ action: 'keep_newer' | 'keep_older' | 'merge' | 'keep_both' | 'manual_review'
224
+ /** Reason for the recommendation */
225
+ reason: string
226
+ /** Confidence in the recommendation (0-1) */
227
+ confidence: number
228
+ }
229
+
230
+ // ============================================================================
231
+ // Strategy Types (Simplified - kept for backwards compatibility in metadata)
232
+ // ============================================================================
233
+
234
+ /**
235
+ * Detection strategy identifiers used in relationship metadata
236
+ * Note: The strategy pattern has been removed. These are kept for metadata tracking only.
237
+ */
238
+ export type DetectionStrategyType =
239
+ | 'similarity' // Pure vector similarity
240
+ | 'temporal' // Time-based relationship inference
241
+ | 'entityOverlap' // Shared entity detection
242
+ | 'llmVerification' // LLM-based classification
243
+ | 'hybrid' // Combined approaches
244
+
245
+ // ============================================================================
246
+ // Vector Store Interface
247
+ // ============================================================================
248
+
249
+ /**
250
+ * Vector search result for relationship detection
251
+ */
252
+ export interface VectorSearchResult {
253
+ /** Memory ID */
254
+ memoryId: string
255
+ /** Memory object */
256
+ memory: Memory
257
+ /** Similarity score */
258
+ similarity: number
259
+ }
260
+
261
+ /**
262
+ * Interface for vector store operations needed by relationship detector
263
+ */
264
+ export interface VectorStore {
265
+ /**
266
+ * Find similar memories using vector search
267
+ * @param embedding - Query embedding vector
268
+ * @param limit - Maximum results to return
269
+ * @param threshold - Minimum similarity threshold
270
+ * @param filters - Optional metadata filters
271
+ */
272
+ findSimilar(
273
+ embedding: number[],
274
+ limit: number,
275
+ threshold: number,
276
+ filters?: {
277
+ containerTag?: string
278
+ excludeIds?: string[]
279
+ }
280
+ ): Promise<VectorSearchResult[]>
281
+ }
282
+
283
+ // ============================================================================
284
+ // LLM Provider Interface
285
+ // ============================================================================
286
+
287
+ /**
288
+ * LLM verification request
289
+ */
290
+ export interface LLMVerificationRequest {
291
+ /** The new memory content */
292
+ newContent: string
293
+ /** The existing memory content */
294
+ existingContent: string
295
+ /** Detected relationship type */
296
+ proposedType: RelationshipType
297
+ /** Current confidence score */
298
+ currentConfidence: number
299
+ }
300
+
301
+ /**
302
+ * LLM verification response
303
+ */
304
+ export interface LLMVerificationResponse {
305
+ /** Verified relationship type (may differ from proposed) */
306
+ relationshipType: RelationshipType | null
307
+ /** Confidence in the verification (0-1) */
308
+ confidence: number
309
+ /** Explanation of the decision */
310
+ explanation: string
311
+ /** Whether a contradiction was detected */
312
+ isContradiction: boolean
313
+ /** Contradiction details if detected */
314
+ contradictionDetails?: {
315
+ type: ContradictionType
316
+ description: string
317
+ }
318
+ }
319
+
320
+ /**
321
+ * Interface for LLM provider used in verification
322
+ */
323
+ export interface LLMProvider {
324
+ /**
325
+ * Verify a proposed relationship using LLM
326
+ */
327
+ verifyRelationship(request: LLMVerificationRequest): Promise<LLMVerificationResponse>
328
+
329
+ /**
330
+ * Check if two memories contradict each other
331
+ */
332
+ checkContradiction(
333
+ content1: string,
334
+ content2: string
335
+ ): Promise<{
336
+ isContradiction: boolean
337
+ type?: ContradictionType
338
+ confidence: number
339
+ description: string
340
+ }>
341
+ }
342
+
343
+ // ============================================================================
344
+ // Cache Types
345
+ // ============================================================================
346
+
347
+ /**
348
+ * Cached relationship score
349
+ */
350
+ export interface CachedRelationshipScore {
351
+ /** Source memory ID */
352
+ sourceId: string
353
+ /** Target memory ID */
354
+ targetId: string
355
+ /** Cached score */
356
+ score: number
357
+ /** Detected relationship type */
358
+ type: RelationshipType | null
359
+ /** When the cache entry was created */
360
+ cachedAt: number
361
+ }
362
+
363
+ /**
364
+ * Cache key generator
365
+ */
366
+ export function generateCacheKey(sourceId: string, targetId: string): string {
367
+ // Ensure consistent ordering for bidirectional relationships
368
+ const [first, second] = [sourceId, targetId].sort()
369
+ return `${first}:${second}`
370
+ }