@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,313 @@
1
+ /**
2
+ * Logging Infrastructure for Supermemory Clone
3
+ *
4
+ * Provides structured logging with levels (debug, info, warn, error)
5
+ * and support for context and request tracing.
6
+ */
7
+
8
+ /**
9
+ * Log levels in order of severity
10
+ */
11
+ export const LogLevel = {
12
+ DEBUG: 0,
13
+ INFO: 1,
14
+ WARN: 2,
15
+ ERROR: 3,
16
+ SILENT: 4,
17
+ } as const
18
+
19
+ export type LogLevelName = keyof typeof LogLevel
20
+ export type LogLevelValue = (typeof LogLevel)[LogLevelName]
21
+
22
+ /**
23
+ * Log entry structure
24
+ */
25
+ export interface LogEntry {
26
+ /** Log level */
27
+ level: LogLevelName
28
+ /** Log message */
29
+ message: string
30
+ /** Timestamp */
31
+ timestamp: Date
32
+ /** Optional context data */
33
+ context?: Record<string, unknown>
34
+ /** Optional request/trace ID */
35
+ traceId?: string
36
+ /** Error object if applicable */
37
+ error?: Error
38
+ /** Service or module name */
39
+ service?: string
40
+ }
41
+
42
+ /**
43
+ * Logger configuration
44
+ */
45
+ export interface LoggerConfig {
46
+ /** Minimum log level to output */
47
+ level: LogLevelName
48
+ /** Service name for log entries */
49
+ service?: string
50
+ /** Whether to include timestamps */
51
+ includeTimestamp?: boolean
52
+ /** Whether to output as JSON */
53
+ jsonOutput?: boolean
54
+ /** Custom output handler */
55
+ output?: (entry: LogEntry) => void
56
+ }
57
+
58
+ /**
59
+ * Default logger configuration
60
+ */
61
+ const DEFAULT_CONFIG: LoggerConfig = {
62
+ level: (process.env.LOG_LEVEL as LogLevelName) || 'INFO',
63
+ includeTimestamp: true,
64
+ jsonOutput: process.env.NODE_ENV === 'production',
65
+ }
66
+
67
+ /**
68
+ * Format log entry for console output
69
+ */
70
+ function formatLogEntry(entry: LogEntry, config: LoggerConfig): string {
71
+ if (config.jsonOutput) {
72
+ return JSON.stringify({
73
+ level: entry.level,
74
+ message: entry.message,
75
+ timestamp: entry.timestamp.toISOString(),
76
+ service: entry.service,
77
+ traceId: entry.traceId,
78
+ context: entry.context,
79
+ error: entry.error
80
+ ? {
81
+ name: entry.error.name,
82
+ message: entry.error.message,
83
+ stack: entry.error.stack,
84
+ }
85
+ : undefined,
86
+ })
87
+ }
88
+
89
+ const parts: string[] = []
90
+
91
+ if (config.includeTimestamp) {
92
+ parts.push(`[${entry.timestamp.toISOString()}]`)
93
+ }
94
+
95
+ parts.push(`[${entry.level}]`)
96
+
97
+ if (entry.service) {
98
+ parts.push(`[${entry.service}]`)
99
+ }
100
+
101
+ if (entry.traceId) {
102
+ parts.push(`[${entry.traceId}]`)
103
+ }
104
+
105
+ parts.push(entry.message)
106
+
107
+ if (entry.context && Object.keys(entry.context).length > 0) {
108
+ parts.push(JSON.stringify(entry.context))
109
+ }
110
+
111
+ if (entry.error) {
112
+ parts.push(`\n Error: ${entry.error.message}`)
113
+ if (entry.error.stack) {
114
+ parts.push(`\n Stack: ${entry.error.stack}`)
115
+ }
116
+ }
117
+
118
+ return parts.join(' ')
119
+ }
120
+
121
+ /**
122
+ * Default output handler
123
+ */
124
+ function defaultOutput(entry: LogEntry, config: LoggerConfig): void {
125
+ const formatted = formatLogEntry(entry, config)
126
+
127
+ switch (entry.level) {
128
+ case 'ERROR':
129
+ console.error(formatted)
130
+ break
131
+ case 'WARN':
132
+ console.warn(formatted)
133
+ break
134
+ case 'DEBUG':
135
+ console.debug(formatted)
136
+ break
137
+ default:
138
+ console.log(formatted)
139
+ }
140
+ }
141
+
142
+ /**
143
+ * Logger class providing structured logging with levels and context
144
+ */
145
+ export class Logger {
146
+ private config: LoggerConfig
147
+ private traceId?: string
148
+ private defaultContext: Record<string, unknown> = {}
149
+
150
+ constructor(config: Partial<LoggerConfig> = {}) {
151
+ this.config = { ...DEFAULT_CONFIG, ...config }
152
+ }
153
+
154
+ /**
155
+ * Set trace ID for request correlation
156
+ */
157
+ setTraceId(traceId: string): void {
158
+ this.traceId = traceId
159
+ }
160
+
161
+ /**
162
+ * Clear trace ID
163
+ */
164
+ clearTraceId(): void {
165
+ this.traceId = undefined
166
+ }
167
+
168
+ /**
169
+ * Set default context that will be included in all log entries
170
+ */
171
+ setDefaultContext(context: Record<string, unknown>): void {
172
+ this.defaultContext = { ...context }
173
+ }
174
+
175
+ /**
176
+ * Add to default context
177
+ */
178
+ addContext(context: Record<string, unknown>): void {
179
+ this.defaultContext = { ...this.defaultContext, ...context }
180
+ }
181
+
182
+ /**
183
+ * Create a child logger with additional context
184
+ */
185
+ child(context: Record<string, unknown>, service?: string): Logger {
186
+ const child = new Logger({
187
+ ...this.config,
188
+ service: service ?? this.config.service,
189
+ })
190
+ child.traceId = this.traceId
191
+ child.defaultContext = { ...this.defaultContext, ...context }
192
+ return child
193
+ }
194
+
195
+ /**
196
+ * Check if a log level should be output
197
+ */
198
+ private shouldLog(level: LogLevelName): boolean {
199
+ return LogLevel[level] >= LogLevel[this.config.level]
200
+ }
201
+
202
+ /**
203
+ * Create and output a log entry
204
+ */
205
+ private log(level: LogLevelName, message: string, context?: Record<string, unknown>, error?: Error): void {
206
+ if (!this.shouldLog(level)) {
207
+ return
208
+ }
209
+
210
+ const entry: LogEntry = {
211
+ level,
212
+ message,
213
+ timestamp: new Date(),
214
+ context: { ...this.defaultContext, ...context },
215
+ traceId: this.traceId,
216
+ error,
217
+ service: this.config.service,
218
+ }
219
+
220
+ if (this.config.output) {
221
+ this.config.output(entry)
222
+ } else {
223
+ defaultOutput(entry, this.config)
224
+ }
225
+ }
226
+
227
+ /**
228
+ * Log debug message
229
+ */
230
+ debug(message: string, context?: Record<string, unknown>): void {
231
+ this.log('DEBUG', message, context)
232
+ }
233
+
234
+ /**
235
+ * Log info message
236
+ */
237
+ info(message: string, context?: Record<string, unknown>): void {
238
+ this.log('INFO', message, context)
239
+ }
240
+
241
+ /**
242
+ * Log warning message
243
+ */
244
+ warn(message: string, context?: Record<string, unknown>, error?: Error): void {
245
+ this.log('WARN', message, context, error)
246
+ }
247
+
248
+ /**
249
+ * Log error message
250
+ */
251
+ error(message: string, context?: Record<string, unknown>, error?: Error): void {
252
+ this.log('ERROR', message, context, error)
253
+ }
254
+
255
+ /**
256
+ * Log error with automatic error extraction
257
+ */
258
+ errorWithException(message: string, err: unknown, context?: Record<string, unknown>): void {
259
+ const error = err instanceof Error ? err : new Error(String(err))
260
+ this.log('ERROR', message, context, error)
261
+ }
262
+
263
+ /**
264
+ * Create a timing helper for measuring operation duration
265
+ */
266
+ time(label: string): () => void {
267
+ const start = Date.now()
268
+ return () => {
269
+ const duration = Date.now() - start
270
+ this.debug(`${label} completed`, { durationMs: duration })
271
+ }
272
+ }
273
+ }
274
+
275
+ // ============================================================================
276
+ // Factory Functions and Singletons
277
+ // ============================================================================
278
+
279
+ /**
280
+ * Create a new logger instance
281
+ */
282
+ export function createLogger(config?: Partial<LoggerConfig>): Logger {
283
+ return new Logger(config)
284
+ }
285
+
286
+ /**
287
+ * Service-specific loggers
288
+ */
289
+ const loggers = new Map<string, Logger>()
290
+
291
+ /**
292
+ * Get or create a logger for a specific service
293
+ */
294
+ export function getLogger(service: string): Logger {
295
+ let logger = loggers.get(service)
296
+ if (!logger) {
297
+ logger = new Logger({ service })
298
+ loggers.set(service, logger)
299
+ }
300
+ return logger
301
+ }
302
+
303
+ /**
304
+ * Reset all loggers (useful for testing)
305
+ */
306
+ export function resetLoggers(): void {
307
+ loggers.clear()
308
+ }
309
+
310
+ /**
311
+ * Default application logger
312
+ */
313
+ export const logger = createLogger({ service: 'supermemory' })