@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,270 @@
1
+ /**
2
+ * Profile Types for Supermemory Clone
3
+ *
4
+ * User profiles store extracted facts that complement search results.
5
+ * Facts are classified as static (long-term) or dynamic (temporary).
6
+ *
7
+ * Note: Base types are defined in types/profile.base.ts to prevent duplication.
8
+ * This file extends those base types with service-specific fields.
9
+ */
10
+
11
+ import { FactType as BaseFactType, FactSemanticCategory, BaseProfileFact } from '../types/profile.base.js'
12
+
13
+ /**
14
+ * Type of fact - determines lifecycle and expiration behavior
15
+ * Re-exported from base types for convenience
16
+ */
17
+ export type FactType = BaseFactType
18
+
19
+ /**
20
+ * Classification result from the fact classifier
21
+ */
22
+ export interface FactClassification {
23
+ type: FactType
24
+ confidence: number
25
+ reason: string
26
+ suggestedExpirationHours?: number
27
+ }
28
+
29
+ /**
30
+ * A single fact extracted from user content
31
+ * Extends BaseProfileFact with service-specific fields
32
+ */
33
+ export interface ProfileFact extends Omit<BaseProfileFact, 'createdAt' | 'updatedAt'> {
34
+ /** Whether this is a static (long-term) or dynamic (temporary) fact */
35
+ type: FactType
36
+
37
+ /** When this fact was extracted */
38
+ extractedAt: Date
39
+
40
+ /** When this fact expires (only for dynamic facts) */
41
+ expiresAt?: Date
42
+
43
+ /** Category of the fact for organization */
44
+ category?: FactCategory
45
+
46
+ /** Number of times this fact has been reinforced */
47
+ reinforcementCount: number
48
+
49
+ /** Last time this fact was accessed or reinforced */
50
+ lastAccessedAt: Date
51
+ }
52
+
53
+ /**
54
+ * Categories for organizing profile facts (semantic categories)
55
+ *
56
+ * Note: This is distinct from FactCategory in types/index.ts which
57
+ * represents fact lifecycle (static/dynamic/inferred). This type
58
+ * represents the semantic category of what the fact is about.
59
+ *
60
+ * Re-exported from base types for convenience
61
+ */
62
+ export type FactCategory = FactSemanticCategory
63
+
64
+ /**
65
+ * User profile containing all extracted facts
66
+ */
67
+ export interface UserProfile {
68
+ /** Unique identifier for the user/container */
69
+ containerTag: string
70
+
71
+ /** Static facts - long-term, rarely change */
72
+ staticFacts: ProfileFact[]
73
+
74
+ /** Dynamic facts - temporary, expire over time */
75
+ dynamicFacts: ProfileFact[]
76
+
77
+ /** When the profile was created */
78
+ createdAt: Date
79
+
80
+ /** When the profile was last updated */
81
+ updatedAt: Date
82
+
83
+ /** Profile version for optimistic locking */
84
+ version: number
85
+ }
86
+
87
+ /**
88
+ * Options for profile operations
89
+ */
90
+ export interface ProfileOptions {
91
+ /** Whether to auto-extract facts from content */
92
+ autoExtract?: boolean
93
+
94
+ /** Whether to refresh dynamic facts */
95
+ refreshDynamic?: boolean
96
+
97
+ /** Maximum number of dynamic facts to keep */
98
+ maxDynamicFacts?: number
99
+
100
+ /** Default expiration hours for dynamic facts */
101
+ defaultDynamicExpirationHours?: number
102
+
103
+ /**
104
+ * Custom patterns for classifying facts as static (long-term).
105
+ * If provided, these will be used instead of the default STATIC_FACT_PATTERNS.
106
+ * Set to empty array to disable static pattern matching.
107
+ */
108
+ staticFactPatterns?: RegExp[]
109
+
110
+ /**
111
+ * Custom patterns for classifying facts as dynamic (temporary).
112
+ * If provided, these will be used instead of the default DYNAMIC_FACT_PATTERNS.
113
+ * Set to empty array to disable dynamic pattern matching.
114
+ */
115
+ dynamicFactPatterns?: RegExp[]
116
+ }
117
+
118
+ /**
119
+ * Result of fact extraction
120
+ */
121
+ export interface ExtractionResult {
122
+ facts: ProfileFact[]
123
+ rawContent: string
124
+ extractedAt: Date
125
+ processingTimeMs: number
126
+ }
127
+
128
+ /**
129
+ * Criteria for promoting dynamic facts to static
130
+ */
131
+ export interface PromotionCriteria {
132
+ /** Minimum reinforcement count to consider promotion */
133
+ minReinforcementCount: number
134
+
135
+ /** Minimum age in days before considering promotion */
136
+ minAgeDays: number
137
+
138
+ /** Minimum confidence score */
139
+ minConfidence: number
140
+ }
141
+
142
+ /**
143
+ * Default configuration values
144
+ */
145
+ export const PROFILE_DEFAULTS = {
146
+ maxDynamicFacts: 50,
147
+ defaultDynamicExpirationHours: 72, // 3 days
148
+ promotionCriteria: {
149
+ minReinforcementCount: 3,
150
+ minAgeDays: 7,
151
+ minConfidence: 0.8,
152
+ } as PromotionCriteria,
153
+ } as const
154
+
155
+ // ============================================================================
156
+ // Fact Classification Patterns
157
+ // ============================================================================
158
+
159
+ /**
160
+ * Default patterns for classifying facts as static (long-term, rarely change).
161
+ * These patterns match content that typically represents enduring information
162
+ * about a person such as their job, education, location, or skills.
163
+ *
164
+ * Can be overridden via ProfileOptions.staticFactPatterns or
165
+ * the SUPERMEMORY_STATIC_PATTERNS environment variable (JSON array of pattern strings).
166
+ *
167
+ * @example "John is a senior software engineer" - matches job title pattern
168
+ * @example "She has 10 years of experience" - matches experience pattern
169
+ * @example "He graduated from MIT" - matches education pattern
170
+ */
171
+ export const STATIC_FACT_PATTERNS: readonly RegExp[] = [
172
+ /** Matches job titles with optional seniority and specialty prefixes */
173
+ /\b(is|works as|employed as|serves as)\s+(a|an|the)?\s*(senior|junior|lead|principal|staff)?\s*(software|data|machine learning|devops|cloud|frontend|backend|full[\s-]?stack)?\s*(engineer|developer|architect|manager|designer|analyst)/i,
174
+ /** Matches employment statements: works at, employed by, works for, joined */
175
+ /\b(works at|employed by|works for|joined)\s+\w+/i,
176
+ /** Matches education statements: graduated from, studied at, has a degree in */
177
+ /\b(graduated from|studied at|has a degree in|majored in)/i,
178
+ /** Matches location statements: lives in, based in, located in, resides in */
179
+ /\b(lives in|based in|located in|resides in)/i,
180
+ /** Matches preference statements: prefers, always uses, favorite, loves, hates */
181
+ /\b(prefers|always uses|favorite|loves|hates)\s+\w+/i,
182
+ /** Matches experience statements: has X years of experience */
183
+ /\b(has|have)\s+\d+\s+years?\s+(of\s+)?experience/i,
184
+ /** Matches expertise statements: specializes in, expert in, proficient in */
185
+ /\b(specializes in|expert in|proficient in|skilled in)/i,
186
+ /** Matches language ability: speaks, fluent in */
187
+ /\b(speaks|fluent in)\s+\w+/i,
188
+ ] as const
189
+
190
+ /**
191
+ * Default patterns for classifying facts as dynamic (temporary, time-bound).
192
+ * These patterns match content that typically represents current activities,
193
+ * recent events, or plans that will become outdated.
194
+ *
195
+ * Can be overridden via ProfileOptions.dynamicFactPatterns or
196
+ * the SUPERMEMORY_DYNAMIC_PATTERNS environment variable (JSON array of pattern strings).
197
+ *
198
+ * @example "I'm currently working on the auth module" - matches temporal + activity
199
+ * @example "Just finished debugging the API" - matches recency indicator
200
+ * @example "Planning to refactor next week" - matches future intent
201
+ */
202
+ export const DYNAMIC_FACT_PATTERNS: readonly RegExp[] = [
203
+ /** Matches present-moment indicators: currently, right now, today, this week */
204
+ /\b(currently|right now|at the moment|today|this week|this month)/i,
205
+ /** Matches ongoing activity verbs: working on, debugging, implementing, building */
206
+ /\b(working on|debugging|fixing|implementing|building|testing)/i,
207
+ /** Matches recency indicators: just, recently, lately, in the last, past few */
208
+ /\b(just|recently|lately|in the last|past few)/i,
209
+ /** Matches active interaction: meeting with, talking to, discussing with */
210
+ /\b(meeting with|talking to|discussing with)/i,
211
+ /** Matches future intent: planning to, going to, about to, will be */
212
+ /\b(planning to|going to|about to|will be)/i,
213
+ /** Matches difficulty/blockers: struggling with, having trouble with, stuck on */
214
+ /\b(struggling with|having trouble with|stuck on)/i,
215
+ /** Matches investigation: looking into, investigating, researching */
216
+ /\b(looking into|investigating|researching)/i,
217
+ ] as const
218
+
219
+ /**
220
+ * Get fact patterns from environment variable or return defaults.
221
+ * Environment variables should contain JSON arrays of pattern strings.
222
+ *
223
+ * @param envKey - Environment variable name to check
224
+ * @param defaults - Default patterns to use if env var is not set
225
+ * @returns Array of RegExp patterns
226
+ *
227
+ * @example
228
+ * ```typescript
229
+ * // Set via environment variable:
230
+ * // SUPERMEMORY_STATIC_PATTERNS='["\b(is|works as)\b"]'
231
+ * const patterns = getFactPatternsFromEnv('SUPERMEMORY_STATIC_PATTERNS', STATIC_FACT_PATTERNS);
232
+ * ```
233
+ */
234
+ export function getFactPatternsFromEnv(envKey: string, defaults: readonly RegExp[]): RegExp[] {
235
+ const envValue = typeof process !== 'undefined' ? process.env[envKey] : undefined
236
+
237
+ if (!envValue) {
238
+ return [...defaults]
239
+ }
240
+
241
+ try {
242
+ const patternStrings = JSON.parse(envValue) as string[]
243
+ return patternStrings.map((p) => new RegExp(p, 'i'))
244
+ } catch {
245
+ // If parsing fails, return defaults
246
+ return [...defaults]
247
+ }
248
+ }
249
+
250
+ /**
251
+ * Get static fact patterns, checking environment override first.
252
+ * Uses SUPERMEMORY_STATIC_PATTERNS environment variable.
253
+ */
254
+ export function getStaticFactPatterns(customPatterns?: RegExp[]): RegExp[] {
255
+ if (customPatterns !== undefined) {
256
+ return customPatterns
257
+ }
258
+ return getFactPatternsFromEnv('SUPERMEMORY_STATIC_PATTERNS', STATIC_FACT_PATTERNS)
259
+ }
260
+
261
+ /**
262
+ * Get dynamic fact patterns, checking environment override first.
263
+ * Uses SUPERMEMORY_DYNAMIC_PATTERNS environment variable.
264
+ */
265
+ export function getDynamicFactPatterns(customPatterns?: RegExp[]): RegExp[] {
266
+ if (customPatterns !== undefined) {
267
+ return customPatterns
268
+ }
269
+ return getFactPatternsFromEnv('SUPERMEMORY_DYNAMIC_PATTERNS', DYNAMIC_FACT_PATTERNS)
270
+ }