@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,313 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Logging Infrastructure for Supermemory Clone
|
|
3
|
+
*
|
|
4
|
+
* Provides structured logging with levels (debug, info, warn, error)
|
|
5
|
+
* and support for context and request tracing.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Log levels in order of severity
|
|
10
|
+
*/
|
|
11
|
+
export const LogLevel = {
|
|
12
|
+
DEBUG: 0,
|
|
13
|
+
INFO: 1,
|
|
14
|
+
WARN: 2,
|
|
15
|
+
ERROR: 3,
|
|
16
|
+
SILENT: 4,
|
|
17
|
+
} as const
|
|
18
|
+
|
|
19
|
+
export type LogLevelName = keyof typeof LogLevel
|
|
20
|
+
export type LogLevelValue = (typeof LogLevel)[LogLevelName]
|
|
21
|
+
|
|
22
|
+
/**
|
|
23
|
+
* Log entry structure
|
|
24
|
+
*/
|
|
25
|
+
export interface LogEntry {
|
|
26
|
+
/** Log level */
|
|
27
|
+
level: LogLevelName
|
|
28
|
+
/** Log message */
|
|
29
|
+
message: string
|
|
30
|
+
/** Timestamp */
|
|
31
|
+
timestamp: Date
|
|
32
|
+
/** Optional context data */
|
|
33
|
+
context?: Record<string, unknown>
|
|
34
|
+
/** Optional request/trace ID */
|
|
35
|
+
traceId?: string
|
|
36
|
+
/** Error object if applicable */
|
|
37
|
+
error?: Error
|
|
38
|
+
/** Service or module name */
|
|
39
|
+
service?: string
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
/**
|
|
43
|
+
* Logger configuration
|
|
44
|
+
*/
|
|
45
|
+
export interface LoggerConfig {
|
|
46
|
+
/** Minimum log level to output */
|
|
47
|
+
level: LogLevelName
|
|
48
|
+
/** Service name for log entries */
|
|
49
|
+
service?: string
|
|
50
|
+
/** Whether to include timestamps */
|
|
51
|
+
includeTimestamp?: boolean
|
|
52
|
+
/** Whether to output as JSON */
|
|
53
|
+
jsonOutput?: boolean
|
|
54
|
+
/** Custom output handler */
|
|
55
|
+
output?: (entry: LogEntry) => void
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Default logger configuration
|
|
60
|
+
*/
|
|
61
|
+
const DEFAULT_CONFIG: LoggerConfig = {
|
|
62
|
+
level: (process.env.LOG_LEVEL as LogLevelName) || 'INFO',
|
|
63
|
+
includeTimestamp: true,
|
|
64
|
+
jsonOutput: process.env.NODE_ENV === 'production',
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
/**
|
|
68
|
+
* Format log entry for console output
|
|
69
|
+
*/
|
|
70
|
+
function formatLogEntry(entry: LogEntry, config: LoggerConfig): string {
|
|
71
|
+
if (config.jsonOutput) {
|
|
72
|
+
return JSON.stringify({
|
|
73
|
+
level: entry.level,
|
|
74
|
+
message: entry.message,
|
|
75
|
+
timestamp: entry.timestamp.toISOString(),
|
|
76
|
+
service: entry.service,
|
|
77
|
+
traceId: entry.traceId,
|
|
78
|
+
context: entry.context,
|
|
79
|
+
error: entry.error
|
|
80
|
+
? {
|
|
81
|
+
name: entry.error.name,
|
|
82
|
+
message: entry.error.message,
|
|
83
|
+
stack: entry.error.stack,
|
|
84
|
+
}
|
|
85
|
+
: undefined,
|
|
86
|
+
})
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
const parts: string[] = []
|
|
90
|
+
|
|
91
|
+
if (config.includeTimestamp) {
|
|
92
|
+
parts.push(`[${entry.timestamp.toISOString()}]`)
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
parts.push(`[${entry.level}]`)
|
|
96
|
+
|
|
97
|
+
if (entry.service) {
|
|
98
|
+
parts.push(`[${entry.service}]`)
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
if (entry.traceId) {
|
|
102
|
+
parts.push(`[${entry.traceId}]`)
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
parts.push(entry.message)
|
|
106
|
+
|
|
107
|
+
if (entry.context && Object.keys(entry.context).length > 0) {
|
|
108
|
+
parts.push(JSON.stringify(entry.context))
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
if (entry.error) {
|
|
112
|
+
parts.push(`\n Error: ${entry.error.message}`)
|
|
113
|
+
if (entry.error.stack) {
|
|
114
|
+
parts.push(`\n Stack: ${entry.error.stack}`)
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
return parts.join(' ')
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Default output handler
|
|
123
|
+
*/
|
|
124
|
+
function defaultOutput(entry: LogEntry, config: LoggerConfig): void {
|
|
125
|
+
const formatted = formatLogEntry(entry, config)
|
|
126
|
+
|
|
127
|
+
switch (entry.level) {
|
|
128
|
+
case 'ERROR':
|
|
129
|
+
console.error(formatted)
|
|
130
|
+
break
|
|
131
|
+
case 'WARN':
|
|
132
|
+
console.warn(formatted)
|
|
133
|
+
break
|
|
134
|
+
case 'DEBUG':
|
|
135
|
+
console.debug(formatted)
|
|
136
|
+
break
|
|
137
|
+
default:
|
|
138
|
+
console.log(formatted)
|
|
139
|
+
}
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* Logger class providing structured logging with levels and context
|
|
144
|
+
*/
|
|
145
|
+
export class Logger {
|
|
146
|
+
private config: LoggerConfig
|
|
147
|
+
private traceId?: string
|
|
148
|
+
private defaultContext: Record<string, unknown> = {}
|
|
149
|
+
|
|
150
|
+
constructor(config: Partial<LoggerConfig> = {}) {
|
|
151
|
+
this.config = { ...DEFAULT_CONFIG, ...config }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Set trace ID for request correlation
|
|
156
|
+
*/
|
|
157
|
+
setTraceId(traceId: string): void {
|
|
158
|
+
this.traceId = traceId
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Clear trace ID
|
|
163
|
+
*/
|
|
164
|
+
clearTraceId(): void {
|
|
165
|
+
this.traceId = undefined
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
/**
|
|
169
|
+
* Set default context that will be included in all log entries
|
|
170
|
+
*/
|
|
171
|
+
setDefaultContext(context: Record<string, unknown>): void {
|
|
172
|
+
this.defaultContext = { ...context }
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Add to default context
|
|
177
|
+
*/
|
|
178
|
+
addContext(context: Record<string, unknown>): void {
|
|
179
|
+
this.defaultContext = { ...this.defaultContext, ...context }
|
|
180
|
+
}
|
|
181
|
+
|
|
182
|
+
/**
|
|
183
|
+
* Create a child logger with additional context
|
|
184
|
+
*/
|
|
185
|
+
child(context: Record<string, unknown>, service?: string): Logger {
|
|
186
|
+
const child = new Logger({
|
|
187
|
+
...this.config,
|
|
188
|
+
service: service ?? this.config.service,
|
|
189
|
+
})
|
|
190
|
+
child.traceId = this.traceId
|
|
191
|
+
child.defaultContext = { ...this.defaultContext, ...context }
|
|
192
|
+
return child
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
/**
|
|
196
|
+
* Check if a log level should be output
|
|
197
|
+
*/
|
|
198
|
+
private shouldLog(level: LogLevelName): boolean {
|
|
199
|
+
return LogLevel[level] >= LogLevel[this.config.level]
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
/**
|
|
203
|
+
* Create and output a log entry
|
|
204
|
+
*/
|
|
205
|
+
private log(level: LogLevelName, message: string, context?: Record<string, unknown>, error?: Error): void {
|
|
206
|
+
if (!this.shouldLog(level)) {
|
|
207
|
+
return
|
|
208
|
+
}
|
|
209
|
+
|
|
210
|
+
const entry: LogEntry = {
|
|
211
|
+
level,
|
|
212
|
+
message,
|
|
213
|
+
timestamp: new Date(),
|
|
214
|
+
context: { ...this.defaultContext, ...context },
|
|
215
|
+
traceId: this.traceId,
|
|
216
|
+
error,
|
|
217
|
+
service: this.config.service,
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
if (this.config.output) {
|
|
221
|
+
this.config.output(entry)
|
|
222
|
+
} else {
|
|
223
|
+
defaultOutput(entry, this.config)
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
/**
|
|
228
|
+
* Log debug message
|
|
229
|
+
*/
|
|
230
|
+
debug(message: string, context?: Record<string, unknown>): void {
|
|
231
|
+
this.log('DEBUG', message, context)
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/**
|
|
235
|
+
* Log info message
|
|
236
|
+
*/
|
|
237
|
+
info(message: string, context?: Record<string, unknown>): void {
|
|
238
|
+
this.log('INFO', message, context)
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/**
|
|
242
|
+
* Log warning message
|
|
243
|
+
*/
|
|
244
|
+
warn(message: string, context?: Record<string, unknown>, error?: Error): void {
|
|
245
|
+
this.log('WARN', message, context, error)
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/**
|
|
249
|
+
* Log error message
|
|
250
|
+
*/
|
|
251
|
+
error(message: string, context?: Record<string, unknown>, error?: Error): void {
|
|
252
|
+
this.log('ERROR', message, context, error)
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
/**
|
|
256
|
+
* Log error with automatic error extraction
|
|
257
|
+
*/
|
|
258
|
+
errorWithException(message: string, err: unknown, context?: Record<string, unknown>): void {
|
|
259
|
+
const error = err instanceof Error ? err : new Error(String(err))
|
|
260
|
+
this.log('ERROR', message, context, error)
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
/**
|
|
264
|
+
* Create a timing helper for measuring operation duration
|
|
265
|
+
*/
|
|
266
|
+
time(label: string): () => void {
|
|
267
|
+
const start = Date.now()
|
|
268
|
+
return () => {
|
|
269
|
+
const duration = Date.now() - start
|
|
270
|
+
this.debug(`${label} completed`, { durationMs: duration })
|
|
271
|
+
}
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
// ============================================================================
|
|
276
|
+
// Factory Functions and Singletons
|
|
277
|
+
// ============================================================================
|
|
278
|
+
|
|
279
|
+
/**
|
|
280
|
+
* Create a new logger instance
|
|
281
|
+
*/
|
|
282
|
+
export function createLogger(config?: Partial<LoggerConfig>): Logger {
|
|
283
|
+
return new Logger(config)
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/**
|
|
287
|
+
* Service-specific loggers
|
|
288
|
+
*/
|
|
289
|
+
const loggers = new Map<string, Logger>()
|
|
290
|
+
|
|
291
|
+
/**
|
|
292
|
+
* Get or create a logger for a specific service
|
|
293
|
+
*/
|
|
294
|
+
export function getLogger(service: string): Logger {
|
|
295
|
+
let logger = loggers.get(service)
|
|
296
|
+
if (!logger) {
|
|
297
|
+
logger = new Logger({ service })
|
|
298
|
+
loggers.set(service, logger)
|
|
299
|
+
}
|
|
300
|
+
return logger
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
/**
|
|
304
|
+
* Reset all loggers (useful for testing)
|
|
305
|
+
*/
|
|
306
|
+
export function resetLoggers(): void {
|
|
307
|
+
loggers.clear()
|
|
308
|
+
}
|
|
309
|
+
|
|
310
|
+
/**
|
|
311
|
+
* Default application logger
|
|
312
|
+
*/
|
|
313
|
+
export const logger = createLogger({ service: 'supermemory' })
|