@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Profile Types for Supermemory Clone
|
|
3
|
+
*
|
|
4
|
+
* User profiles store extracted facts that complement search results.
|
|
5
|
+
* Facts are classified as static (long-term) or dynamic (temporary).
|
|
6
|
+
*
|
|
7
|
+
* Note: Base types are defined in types/profile.base.ts to prevent duplication.
|
|
8
|
+
* This file extends those base types with service-specific fields.
|
|
9
|
+
*/
|
|
10
|
+
|
|
11
|
+
import { FactType as BaseFactType, FactSemanticCategory, BaseProfileFact } from '../types/profile.base.js'
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Type of fact - determines lifecycle and expiration behavior
|
|
15
|
+
* Re-exported from base types for convenience
|
|
16
|
+
*/
|
|
17
|
+
export type FactType = BaseFactType
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Classification result from the fact classifier
|
|
21
|
+
*/
|
|
22
|
+
export interface FactClassification {
|
|
23
|
+
type: FactType
|
|
24
|
+
confidence: number
|
|
25
|
+
reason: string
|
|
26
|
+
suggestedExpirationHours?: number
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* A single fact extracted from user content
|
|
31
|
+
* Extends BaseProfileFact with service-specific fields
|
|
32
|
+
*/
|
|
33
|
+
export interface ProfileFact extends Omit<BaseProfileFact, 'createdAt' | 'updatedAt'> {
|
|
34
|
+
/** Whether this is a static (long-term) or dynamic (temporary) fact */
|
|
35
|
+
type: FactType
|
|
36
|
+
|
|
37
|
+
/** When this fact was extracted */
|
|
38
|
+
extractedAt: Date
|
|
39
|
+
|
|
40
|
+
/** When this fact expires (only for dynamic facts) */
|
|
41
|
+
expiresAt?: Date
|
|
42
|
+
|
|
43
|
+
/** Category of the fact for organization */
|
|
44
|
+
category?: FactCategory
|
|
45
|
+
|
|
46
|
+
/** Number of times this fact has been reinforced */
|
|
47
|
+
reinforcementCount: number
|
|
48
|
+
|
|
49
|
+
/** Last time this fact was accessed or reinforced */
|
|
50
|
+
lastAccessedAt: Date
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/**
|
|
54
|
+
* Categories for organizing profile facts (semantic categories)
|
|
55
|
+
*
|
|
56
|
+
* Note: This is distinct from FactCategory in types/index.ts which
|
|
57
|
+
* represents fact lifecycle (static/dynamic/inferred). This type
|
|
58
|
+
* represents the semantic category of what the fact is about.
|
|
59
|
+
*
|
|
60
|
+
* Re-exported from base types for convenience
|
|
61
|
+
*/
|
|
62
|
+
export type FactCategory = FactSemanticCategory
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* User profile containing all extracted facts
|
|
66
|
+
*/
|
|
67
|
+
export interface UserProfile {
|
|
68
|
+
/** Unique identifier for the user/container */
|
|
69
|
+
containerTag: string
|
|
70
|
+
|
|
71
|
+
/** Static facts - long-term, rarely change */
|
|
72
|
+
staticFacts: ProfileFact[]
|
|
73
|
+
|
|
74
|
+
/** Dynamic facts - temporary, expire over time */
|
|
75
|
+
dynamicFacts: ProfileFact[]
|
|
76
|
+
|
|
77
|
+
/** When the profile was created */
|
|
78
|
+
createdAt: Date
|
|
79
|
+
|
|
80
|
+
/** When the profile was last updated */
|
|
81
|
+
updatedAt: Date
|
|
82
|
+
|
|
83
|
+
/** Profile version for optimistic locking */
|
|
84
|
+
version: number
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Options for profile operations
|
|
89
|
+
*/
|
|
90
|
+
export interface ProfileOptions {
|
|
91
|
+
/** Whether to auto-extract facts from content */
|
|
92
|
+
autoExtract?: boolean
|
|
93
|
+
|
|
94
|
+
/** Whether to refresh dynamic facts */
|
|
95
|
+
refreshDynamic?: boolean
|
|
96
|
+
|
|
97
|
+
/** Maximum number of dynamic facts to keep */
|
|
98
|
+
maxDynamicFacts?: number
|
|
99
|
+
|
|
100
|
+
/** Default expiration hours for dynamic facts */
|
|
101
|
+
defaultDynamicExpirationHours?: number
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Custom patterns for classifying facts as static (long-term).
|
|
105
|
+
* If provided, these will be used instead of the default STATIC_FACT_PATTERNS.
|
|
106
|
+
* Set to empty array to disable static pattern matching.
|
|
107
|
+
*/
|
|
108
|
+
staticFactPatterns?: RegExp[]
|
|
109
|
+
|
|
110
|
+
/**
|
|
111
|
+
* Custom patterns for classifying facts as dynamic (temporary).
|
|
112
|
+
* If provided, these will be used instead of the default DYNAMIC_FACT_PATTERNS.
|
|
113
|
+
* Set to empty array to disable dynamic pattern matching.
|
|
114
|
+
*/
|
|
115
|
+
dynamicFactPatterns?: RegExp[]
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Result of fact extraction
|
|
120
|
+
*/
|
|
121
|
+
export interface ExtractionResult {
|
|
122
|
+
facts: ProfileFact[]
|
|
123
|
+
rawContent: string
|
|
124
|
+
extractedAt: Date
|
|
125
|
+
processingTimeMs: number
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
/**
|
|
129
|
+
* Criteria for promoting dynamic facts to static
|
|
130
|
+
*/
|
|
131
|
+
export interface PromotionCriteria {
|
|
132
|
+
/** Minimum reinforcement count to consider promotion */
|
|
133
|
+
minReinforcementCount: number
|
|
134
|
+
|
|
135
|
+
/** Minimum age in days before considering promotion */
|
|
136
|
+
minAgeDays: number
|
|
137
|
+
|
|
138
|
+
/** Minimum confidence score */
|
|
139
|
+
minConfidence: number
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Default configuration values
|
|
144
|
+
*/
|
|
145
|
+
export const PROFILE_DEFAULTS = {
|
|
146
|
+
maxDynamicFacts: 50,
|
|
147
|
+
defaultDynamicExpirationHours: 72, // 3 days
|
|
148
|
+
promotionCriteria: {
|
|
149
|
+
minReinforcementCount: 3,
|
|
150
|
+
minAgeDays: 7,
|
|
151
|
+
minConfidence: 0.8,
|
|
152
|
+
} as PromotionCriteria,
|
|
153
|
+
} as const
|
|
154
|
+
|
|
155
|
+
// ============================================================================
|
|
156
|
+
// Fact Classification Patterns
|
|
157
|
+
// ============================================================================
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Default patterns for classifying facts as static (long-term, rarely change).
|
|
161
|
+
* These patterns match content that typically represents enduring information
|
|
162
|
+
* about a person such as their job, education, location, or skills.
|
|
163
|
+
*
|
|
164
|
+
* Can be overridden via ProfileOptions.staticFactPatterns or
|
|
165
|
+
* the SUPERMEMORY_STATIC_PATTERNS environment variable (JSON array of pattern strings).
|
|
166
|
+
*
|
|
167
|
+
* @example "John is a senior software engineer" - matches job title pattern
|
|
168
|
+
* @example "She has 10 years of experience" - matches experience pattern
|
|
169
|
+
* @example "He graduated from MIT" - matches education pattern
|
|
170
|
+
*/
|
|
171
|
+
export const STATIC_FACT_PATTERNS: readonly RegExp[] = [
|
|
172
|
+
/** Matches job titles with optional seniority and specialty prefixes */
|
|
173
|
+
/\b(is|works as|employed as|serves as)\s+(a|an|the)?\s*(senior|junior|lead|principal|staff)?\s*(software|data|machine learning|devops|cloud|frontend|backend|full[\s-]?stack)?\s*(engineer|developer|architect|manager|designer|analyst)/i,
|
|
174
|
+
/** Matches employment statements: works at, employed by, works for, joined */
|
|
175
|
+
/\b(works at|employed by|works for|joined)\s+\w+/i,
|
|
176
|
+
/** Matches education statements: graduated from, studied at, has a degree in */
|
|
177
|
+
/\b(graduated from|studied at|has a degree in|majored in)/i,
|
|
178
|
+
/** Matches location statements: lives in, based in, located in, resides in */
|
|
179
|
+
/\b(lives in|based in|located in|resides in)/i,
|
|
180
|
+
/** Matches preference statements: prefers, always uses, favorite, loves, hates */
|
|
181
|
+
/\b(prefers|always uses|favorite|loves|hates)\s+\w+/i,
|
|
182
|
+
/** Matches experience statements: has X years of experience */
|
|
183
|
+
/\b(has|have)\s+\d+\s+years?\s+(of\s+)?experience/i,
|
|
184
|
+
/** Matches expertise statements: specializes in, expert in, proficient in */
|
|
185
|
+
/\b(specializes in|expert in|proficient in|skilled in)/i,
|
|
186
|
+
/** Matches language ability: speaks, fluent in */
|
|
187
|
+
/\b(speaks|fluent in)\s+\w+/i,
|
|
188
|
+
] as const
|
|
189
|
+
|
|
190
|
+
/**
|
|
191
|
+
* Default patterns for classifying facts as dynamic (temporary, time-bound).
|
|
192
|
+
* These patterns match content that typically represents current activities,
|
|
193
|
+
* recent events, or plans that will become outdated.
|
|
194
|
+
*
|
|
195
|
+
* Can be overridden via ProfileOptions.dynamicFactPatterns or
|
|
196
|
+
* the SUPERMEMORY_DYNAMIC_PATTERNS environment variable (JSON array of pattern strings).
|
|
197
|
+
*
|
|
198
|
+
* @example "I'm currently working on the auth module" - matches temporal + activity
|
|
199
|
+
* @example "Just finished debugging the API" - matches recency indicator
|
|
200
|
+
* @example "Planning to refactor next week" - matches future intent
|
|
201
|
+
*/
|
|
202
|
+
export const DYNAMIC_FACT_PATTERNS: readonly RegExp[] = [
|
|
203
|
+
/** Matches present-moment indicators: currently, right now, today, this week */
|
|
204
|
+
/\b(currently|right now|at the moment|today|this week|this month)/i,
|
|
205
|
+
/** Matches ongoing activity verbs: working on, debugging, implementing, building */
|
|
206
|
+
/\b(working on|debugging|fixing|implementing|building|testing)/i,
|
|
207
|
+
/** Matches recency indicators: just, recently, lately, in the last, past few */
|
|
208
|
+
/\b(just|recently|lately|in the last|past few)/i,
|
|
209
|
+
/** Matches active interaction: meeting with, talking to, discussing with */
|
|
210
|
+
/\b(meeting with|talking to|discussing with)/i,
|
|
211
|
+
/** Matches future intent: planning to, going to, about to, will be */
|
|
212
|
+
/\b(planning to|going to|about to|will be)/i,
|
|
213
|
+
/** Matches difficulty/blockers: struggling with, having trouble with, stuck on */
|
|
214
|
+
/\b(struggling with|having trouble with|stuck on)/i,
|
|
215
|
+
/** Matches investigation: looking into, investigating, researching */
|
|
216
|
+
/\b(looking into|investigating|researching)/i,
|
|
217
|
+
] as const
|
|
218
|
+
|
|
219
|
+
/**
|
|
220
|
+
* Get fact patterns from environment variable or return defaults.
|
|
221
|
+
* Environment variables should contain JSON arrays of pattern strings.
|
|
222
|
+
*
|
|
223
|
+
* @param envKey - Environment variable name to check
|
|
224
|
+
* @param defaults - Default patterns to use if env var is not set
|
|
225
|
+
* @returns Array of RegExp patterns
|
|
226
|
+
*
|
|
227
|
+
* @example
|
|
228
|
+
* ```typescript
|
|
229
|
+
* // Set via environment variable:
|
|
230
|
+
* // SUPERMEMORY_STATIC_PATTERNS='["\b(is|works as)\b"]'
|
|
231
|
+
* const patterns = getFactPatternsFromEnv('SUPERMEMORY_STATIC_PATTERNS', STATIC_FACT_PATTERNS);
|
|
232
|
+
* ```
|
|
233
|
+
*/
|
|
234
|
+
export function getFactPatternsFromEnv(envKey: string, defaults: readonly RegExp[]): RegExp[] {
|
|
235
|
+
const envValue = typeof process !== 'undefined' ? process.env[envKey] : undefined
|
|
236
|
+
|
|
237
|
+
if (!envValue) {
|
|
238
|
+
return [...defaults]
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
try {
|
|
242
|
+
const patternStrings = JSON.parse(envValue) as string[]
|
|
243
|
+
return patternStrings.map((p) => new RegExp(p, 'i'))
|
|
244
|
+
} catch {
|
|
245
|
+
// If parsing fails, return defaults
|
|
246
|
+
return [...defaults]
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
/**
|
|
251
|
+
* Get static fact patterns, checking environment override first.
|
|
252
|
+
* Uses SUPERMEMORY_STATIC_PATTERNS environment variable.
|
|
253
|
+
*/
|
|
254
|
+
export function getStaticFactPatterns(customPatterns?: RegExp[]): RegExp[] {
|
|
255
|
+
if (customPatterns !== undefined) {
|
|
256
|
+
return customPatterns
|
|
257
|
+
}
|
|
258
|
+
return getFactPatternsFromEnv('SUPERMEMORY_STATIC_PATTERNS', STATIC_FACT_PATTERNS)
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
/**
|
|
262
|
+
* Get dynamic fact patterns, checking environment override first.
|
|
263
|
+
* Uses SUPERMEMORY_DYNAMIC_PATTERNS environment variable.
|
|
264
|
+
*/
|
|
265
|
+
export function getDynamicFactPatterns(customPatterns?: RegExp[]): RegExp[] {
|
|
266
|
+
if (customPatterns !== undefined) {
|
|
267
|
+
return customPatterns
|
|
268
|
+
}
|
|
269
|
+
return getFactPatternsFromEnv('SUPERMEMORY_DYNAMIC_PATTERNS', DYNAMIC_FACT_PATTERNS)
|
|
270
|
+
}
|