@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,403 @@
1
+ /**
2
+ * Mock Vector Store
3
+ *
4
+ * A configurable mock implementation for testing.
5
+ * Allows simulating various behaviors including errors,
6
+ * delays, and specific search results.
7
+ */
8
+
9
+ import {
10
+ VectorEntry,
11
+ VectorSearchResult,
12
+ SearchOptions,
13
+ AddOptions,
14
+ DeleteOptions,
15
+ VectorStoreConfig,
16
+ VectorStoreStats,
17
+ BatchResult,
18
+ } from './types.js'
19
+ import { BaseVectorStore, validateVector } from './base.js'
20
+
21
+ /**
22
+ * Mock configuration options
23
+ */
24
+ export interface MockVectorStoreOptions {
25
+ /** Simulate initialization delay in ms */
26
+ initDelay?: number
27
+
28
+ /** Simulate operation delay in ms */
29
+ operationDelay?: number
30
+
31
+ /** Throw error on specific operations */
32
+ failOn?: {
33
+ initialize?: boolean | Error
34
+ add?: boolean | Error
35
+ search?: boolean | Error
36
+ delete?: boolean | Error
37
+ get?: boolean | Error
38
+ }
39
+
40
+ /** Pre-populate with entries */
41
+ initialEntries?: VectorEntry[]
42
+
43
+ /** Fixed search results to return */
44
+ fixedSearchResults?: VectorSearchResult[]
45
+
46
+ /** Record all operations for assertions */
47
+ recordOperations?: boolean
48
+ }
49
+
50
+ /**
51
+ * Recorded operation for testing assertions
52
+ */
53
+ export interface RecordedOperation {
54
+ type: 'initialize' | 'add' | 'addBatch' | 'update' | 'delete' | 'get' | 'exists' | 'search' | 'clear' | 'close'
55
+ timestamp: Date
56
+ args?: unknown
57
+ result?: unknown
58
+ error?: Error
59
+ }
60
+
61
+ /**
62
+ * Mock Vector Store for Testing
63
+ */
64
+ export class MockVectorStore extends BaseVectorStore {
65
+ private entries: Map<string, VectorEntry> = new Map()
66
+ private initialized = false
67
+ private readonly mockOptions: MockVectorStoreOptions
68
+ private operations: RecordedOperation[] = []
69
+
70
+ constructor(config: VectorStoreConfig, mockOptions: MockVectorStoreOptions = {}) {
71
+ super({
72
+ ...config,
73
+ provider: 'memory', // Use memory as base provider type
74
+ })
75
+ this.mockOptions = mockOptions
76
+ }
77
+
78
+ /**
79
+ * Get recorded operations (for test assertions)
80
+ */
81
+ getOperations(): RecordedOperation[] {
82
+ return [...this.operations]
83
+ }
84
+
85
+ /**
86
+ * Clear recorded operations
87
+ */
88
+ clearOperations(): void {
89
+ this.operations = []
90
+ }
91
+
92
+ /**
93
+ * Get last operation of a specific type
94
+ */
95
+ getLastOperation(type: RecordedOperation['type']): RecordedOperation | undefined {
96
+ return [...this.operations].reverse().find((op) => op.type === type)
97
+ }
98
+
99
+ /**
100
+ * Record an operation
101
+ */
102
+ private recordOp(type: RecordedOperation['type'], args?: unknown, result?: unknown, error?: Error): void {
103
+ if (this.mockOptions.recordOperations !== false) {
104
+ this.operations.push({
105
+ type,
106
+ timestamp: new Date(),
107
+ args,
108
+ result,
109
+ error,
110
+ })
111
+ }
112
+ }
113
+
114
+ /**
115
+ * Simulate delay if configured
116
+ */
117
+ private async delay(type: 'init' | 'operation'): Promise<void> {
118
+ const ms = type === 'init' ? this.mockOptions.initDelay : this.mockOptions.operationDelay
119
+
120
+ if (ms && ms > 0) {
121
+ await new Promise((resolve) => setTimeout(resolve, ms))
122
+ }
123
+ }
124
+
125
+ /**
126
+ * Check if should fail and throw if so
127
+ */
128
+ private checkFail(operation: keyof NonNullable<MockVectorStoreOptions['failOn']>): void {
129
+ const failConfig = this.mockOptions.failOn?.[operation]
130
+ if (failConfig) {
131
+ if (failConfig instanceof Error) {
132
+ throw failConfig
133
+ }
134
+ throw new Error(`Mock error: ${operation} failed`)
135
+ }
136
+ }
137
+
138
+ /**
139
+ * Initialize the mock store
140
+ */
141
+ async initialize(): Promise<void> {
142
+ await this.delay('init')
143
+ this.checkFail('initialize')
144
+
145
+ // Populate initial entries
146
+ if (this.mockOptions.initialEntries) {
147
+ for (const entry of this.mockOptions.initialEntries) {
148
+ this.entries.set(entry.id, { ...entry })
149
+ }
150
+ }
151
+
152
+ this.initialized = true
153
+ this.recordOp('initialize')
154
+ }
155
+
156
+ /**
157
+ * Add a single vector entry
158
+ */
159
+ async add(entry: VectorEntry, options?: AddOptions): Promise<void> {
160
+ await this.delay('operation')
161
+ this.checkFail('add')
162
+ this.validateEntry(entry)
163
+
164
+ if (this.entries.has(entry.id) && !options?.overwrite) {
165
+ const error = new Error(`Entry with ID ${entry.id} already exists`)
166
+ this.recordOp('add', { entry, options }, undefined, error)
167
+ throw error
168
+ }
169
+
170
+ this.entries.set(entry.id, {
171
+ ...entry,
172
+ createdAt: entry.createdAt ?? new Date(),
173
+ updatedAt: new Date(),
174
+ })
175
+
176
+ this.recordOp('add', { entry, options })
177
+ this.emit('add', { id: entry.id })
178
+ }
179
+
180
+ /**
181
+ * Add multiple vector entries
182
+ */
183
+ async addBatch(entries: VectorEntry[], options?: AddOptions): Promise<BatchResult> {
184
+ await this.delay('operation')
185
+
186
+ const result: BatchResult = {
187
+ successful: 0,
188
+ failed: 0,
189
+ errors: [],
190
+ }
191
+
192
+ for (const entry of entries) {
193
+ try {
194
+ await this.add(entry, options)
195
+ result.successful++
196
+ } catch (error) {
197
+ result.failed++
198
+ result.errors?.push({
199
+ id: entry.id,
200
+ error: error instanceof Error ? error.message : String(error),
201
+ })
202
+ }
203
+ }
204
+
205
+ this.recordOp('addBatch', { entries, options }, result)
206
+ return result
207
+ }
208
+
209
+ /**
210
+ * Update an existing vector entry
211
+ */
212
+ async update(id: string, updates: Partial<VectorEntry>): Promise<boolean> {
213
+ await this.delay('operation')
214
+
215
+ const existing = this.entries.get(id)
216
+ if (!existing) {
217
+ this.recordOp('update', { id, updates }, false)
218
+ return false
219
+ }
220
+
221
+ if (updates.embedding) {
222
+ validateVector(updates.embedding, this.config.dimensions)
223
+ }
224
+
225
+ this.entries.set(id, {
226
+ ...existing,
227
+ ...updates,
228
+ id, // Ensure ID unchanged
229
+ updatedAt: new Date(),
230
+ })
231
+
232
+ this.recordOp('update', { id, updates }, true)
233
+ this.emit('update', { id })
234
+ return true
235
+ }
236
+
237
+ /**
238
+ * Delete vector entries
239
+ */
240
+ async delete(options: DeleteOptions): Promise<number> {
241
+ await this.delay('operation')
242
+ this.checkFail('delete')
243
+
244
+ let deleted = 0
245
+
246
+ if (options.deleteAll) {
247
+ deleted = this.entries.size
248
+ this.entries.clear()
249
+ } else if (options.ids && options.ids.length > 0) {
250
+ for (const id of options.ids) {
251
+ if (this.entries.delete(id)) {
252
+ deleted++
253
+ }
254
+ }
255
+ } else if (options.filter) {
256
+ for (const [id, entry] of this.entries) {
257
+ if (this.matchesFilter(entry.metadata, options.filter)) {
258
+ this.entries.delete(id)
259
+ deleted++
260
+ }
261
+ }
262
+ }
263
+
264
+ this.recordOp('delete', options, deleted)
265
+
266
+ if (deleted > 0) {
267
+ this.emit('delete', { count: deleted })
268
+ }
269
+
270
+ return deleted
271
+ }
272
+
273
+ /**
274
+ * Get a vector entry by ID
275
+ */
276
+ async get(id: string): Promise<VectorEntry | null> {
277
+ await this.delay('operation')
278
+ this.checkFail('get')
279
+
280
+ const entry = this.entries.get(id) ?? null
281
+ this.recordOp('get', { id }, entry)
282
+ return entry ? { ...entry } : null
283
+ }
284
+
285
+ /**
286
+ * Check if a vector entry exists
287
+ */
288
+ async exists(id: string): Promise<boolean> {
289
+ await this.delay('operation')
290
+
291
+ const exists = this.entries.has(id)
292
+ this.recordOp('exists', { id }, exists)
293
+ return exists
294
+ }
295
+
296
+ /**
297
+ * Search for similar vectors
298
+ */
299
+ async search(query: number[], options?: SearchOptions): Promise<VectorSearchResult[]> {
300
+ await this.delay('operation')
301
+ this.checkFail('search')
302
+ validateVector(query, this.config.dimensions)
303
+
304
+ // Return fixed results if configured
305
+ if (this.mockOptions.fixedSearchResults) {
306
+ this.recordOp('search', { query, options }, this.mockOptions.fixedSearchResults)
307
+ return this.mockOptions.fixedSearchResults
308
+ }
309
+
310
+ const opts = this.mergeOptions(options)
311
+
312
+ // Perform actual similarity search
313
+ let candidates = Array.from(this.entries.values())
314
+ candidates = this.applyFilters(candidates, opts.filters)
315
+
316
+ const results: VectorSearchResult[] = []
317
+ for (const entry of candidates) {
318
+ const score = this.calculateSimilarity(query, entry.embedding)
319
+
320
+ if (score >= opts.threshold) {
321
+ results.push({
322
+ id: entry.id,
323
+ score,
324
+ embedding: opts.includeVectors ? entry.embedding : undefined,
325
+ metadata: opts.includeMetadata ? entry.metadata : {},
326
+ })
327
+ }
328
+ }
329
+
330
+ results.sort((a, b) => b.score - a.score)
331
+ const limited = results.slice(0, opts.limit)
332
+
333
+ this.recordOp('search', { query, options }, limited)
334
+ this.emit('search', { resultsCount: limited.length })
335
+
336
+ return limited
337
+ }
338
+
339
+ /**
340
+ * Get statistics about the vector store
341
+ */
342
+ async getStats(): Promise<VectorStoreStats> {
343
+ return {
344
+ totalVectors: this.entries.size,
345
+ dimensions: this.config.dimensions,
346
+ indexType: 'flat',
347
+ metric: this.config.metric ?? 'cosine',
348
+ indexBuilt: true,
349
+ namespaces: ['default'],
350
+ }
351
+ }
352
+
353
+ /**
354
+ * Clear all vectors from the store
355
+ */
356
+ async clear(): Promise<void> {
357
+ this.entries.clear()
358
+ this.recordOp('clear')
359
+ this.emit('delete', { deleteAll: true })
360
+ }
361
+
362
+ /**
363
+ * Close the vector store
364
+ */
365
+ async close(): Promise<void> {
366
+ this.entries.clear()
367
+ this.initialized = false
368
+ this.recordOp('close')
369
+ }
370
+
371
+ /**
372
+ * Get all entries (for testing)
373
+ */
374
+ async getAllEntries(): Promise<VectorEntry[]> {
375
+ return Array.from(this.entries.values())
376
+ }
377
+
378
+ /**
379
+ * Set entries directly (for test setup)
380
+ */
381
+ setEntries(entries: VectorEntry[]): void {
382
+ this.entries.clear()
383
+ for (const entry of entries) {
384
+ this.entries.set(entry.id, entry)
385
+ }
386
+ }
387
+ }
388
+
389
+ /**
390
+ * Create a mock vector store
391
+ */
392
+ export function createMockVectorStore(
393
+ dimensions: number = 1536,
394
+ mockOptions: MockVectorStoreOptions = {}
395
+ ): MockVectorStore {
396
+ return new MockVectorStore(
397
+ {
398
+ provider: 'memory',
399
+ dimensions,
400
+ },
401
+ mockOptions
402
+ )
403
+ }