@twelvehart/supermemory-runtime 1.0.0-next.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (156) hide show
  1. package/.env.example +57 -0
  2. package/README.md +374 -0
  3. package/dist/index.js +189 -0
  4. package/dist/mcp/index.js +1132 -0
  5. package/docker-compose.prod.yml +91 -0
  6. package/docker-compose.yml +358 -0
  7. package/drizzle/0000_dapper_the_professor.sql +159 -0
  8. package/drizzle/0001_api_keys.sql +51 -0
  9. package/drizzle/meta/0000_snapshot.json +1532 -0
  10. package/drizzle/meta/_journal.json +13 -0
  11. package/drizzle.config.ts +20 -0
  12. package/package.json +114 -0
  13. package/scripts/add-extraction-job.ts +122 -0
  14. package/scripts/benchmark-pgvector.ts +122 -0
  15. package/scripts/bootstrap.sh +209 -0
  16. package/scripts/check-runtime-pack.ts +111 -0
  17. package/scripts/claude-mcp-config.ts +336 -0
  18. package/scripts/docker-entrypoint.sh +183 -0
  19. package/scripts/doctor.ts +377 -0
  20. package/scripts/init-db.sql +33 -0
  21. package/scripts/install.sh +1110 -0
  22. package/scripts/mcp-setup.ts +271 -0
  23. package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
  24. package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
  25. package/scripts/migrations/003_create_hnsw_index.sql +94 -0
  26. package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
  27. package/scripts/migrations/005_create_chunks_table.sql +95 -0
  28. package/scripts/migrations/006_create_processing_queue.sql +45 -0
  29. package/scripts/migrations/generate_test_data.sql +42 -0
  30. package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
  31. package/scripts/migrations/run_migrations.sh +286 -0
  32. package/scripts/migrations/test_hnsw_index.sql +255 -0
  33. package/scripts/pre-commit-secrets +282 -0
  34. package/scripts/run-extraction-worker.ts +46 -0
  35. package/scripts/run-phase1-tests.sh +291 -0
  36. package/scripts/setup.ts +222 -0
  37. package/scripts/smoke-install.sh +12 -0
  38. package/scripts/test-health-endpoint.sh +328 -0
  39. package/src/api/index.ts +2 -0
  40. package/src/api/middleware/auth.ts +80 -0
  41. package/src/api/middleware/csrf.ts +308 -0
  42. package/src/api/middleware/errorHandler.ts +166 -0
  43. package/src/api/middleware/rateLimit.ts +360 -0
  44. package/src/api/middleware/validation.ts +514 -0
  45. package/src/api/routes/documents.ts +286 -0
  46. package/src/api/routes/profiles.ts +237 -0
  47. package/src/api/routes/search.ts +71 -0
  48. package/src/api/stores/index.ts +58 -0
  49. package/src/config/bootstrap-env.ts +3 -0
  50. package/src/config/env.ts +71 -0
  51. package/src/config/feature-flags.ts +25 -0
  52. package/src/config/index.ts +140 -0
  53. package/src/config/secrets.config.ts +291 -0
  54. package/src/db/client.ts +92 -0
  55. package/src/db/index.ts +73 -0
  56. package/src/db/postgres.ts +72 -0
  57. package/src/db/schema/chunks.schema.ts +31 -0
  58. package/src/db/schema/containers.schema.ts +46 -0
  59. package/src/db/schema/documents.schema.ts +49 -0
  60. package/src/db/schema/embeddings.schema.ts +32 -0
  61. package/src/db/schema/index.ts +11 -0
  62. package/src/db/schema/memories.schema.ts +72 -0
  63. package/src/db/schema/profiles.schema.ts +34 -0
  64. package/src/db/schema/queue.schema.ts +59 -0
  65. package/src/db/schema/relationships.schema.ts +42 -0
  66. package/src/db/schema.ts +223 -0
  67. package/src/db/worker-connection.ts +47 -0
  68. package/src/index.ts +235 -0
  69. package/src/mcp/CLAUDE.md +1 -0
  70. package/src/mcp/index.ts +1380 -0
  71. package/src/mcp/legacyState.ts +22 -0
  72. package/src/mcp/rateLimit.ts +358 -0
  73. package/src/mcp/resources.ts +309 -0
  74. package/src/mcp/results.ts +104 -0
  75. package/src/mcp/tools.ts +401 -0
  76. package/src/queues/config.ts +119 -0
  77. package/src/queues/index.ts +289 -0
  78. package/src/sdk/client.ts +225 -0
  79. package/src/sdk/errors.ts +266 -0
  80. package/src/sdk/http.ts +560 -0
  81. package/src/sdk/index.ts +244 -0
  82. package/src/sdk/resources/base.ts +65 -0
  83. package/src/sdk/resources/connections.ts +204 -0
  84. package/src/sdk/resources/documents.ts +163 -0
  85. package/src/sdk/resources/index.ts +10 -0
  86. package/src/sdk/resources/memories.ts +150 -0
  87. package/src/sdk/resources/search.ts +60 -0
  88. package/src/sdk/resources/settings.ts +36 -0
  89. package/src/sdk/types.ts +674 -0
  90. package/src/services/chunking/index.ts +451 -0
  91. package/src/services/chunking.service.ts +650 -0
  92. package/src/services/csrf.service.ts +252 -0
  93. package/src/services/documents.repository.ts +219 -0
  94. package/src/services/documents.service.ts +191 -0
  95. package/src/services/embedding.service.ts +404 -0
  96. package/src/services/extraction.service.ts +300 -0
  97. package/src/services/extractors/code.extractor.ts +451 -0
  98. package/src/services/extractors/index.ts +9 -0
  99. package/src/services/extractors/markdown.extractor.ts +461 -0
  100. package/src/services/extractors/pdf.extractor.ts +315 -0
  101. package/src/services/extractors/text.extractor.ts +118 -0
  102. package/src/services/extractors/url.extractor.ts +243 -0
  103. package/src/services/index.ts +235 -0
  104. package/src/services/ingestion.service.ts +177 -0
  105. package/src/services/llm/anthropic.ts +400 -0
  106. package/src/services/llm/base.ts +460 -0
  107. package/src/services/llm/contradiction-detector.service.ts +526 -0
  108. package/src/services/llm/heuristics.ts +148 -0
  109. package/src/services/llm/index.ts +309 -0
  110. package/src/services/llm/memory-classifier.service.ts +383 -0
  111. package/src/services/llm/memory-extension-detector.service.ts +523 -0
  112. package/src/services/llm/mock.ts +470 -0
  113. package/src/services/llm/openai.ts +398 -0
  114. package/src/services/llm/prompts.ts +438 -0
  115. package/src/services/llm/types.ts +373 -0
  116. package/src/services/memory.repository.ts +1769 -0
  117. package/src/services/memory.service.ts +1338 -0
  118. package/src/services/memory.types.ts +234 -0
  119. package/src/services/persistence/index.ts +295 -0
  120. package/src/services/pipeline.service.ts +509 -0
  121. package/src/services/profile.repository.ts +436 -0
  122. package/src/services/profile.service.ts +560 -0
  123. package/src/services/profile.types.ts +270 -0
  124. package/src/services/relationships/detector.ts +1128 -0
  125. package/src/services/relationships/index.ts +268 -0
  126. package/src/services/relationships/memory-integration.ts +459 -0
  127. package/src/services/relationships/strategies.ts +132 -0
  128. package/src/services/relationships/types.ts +370 -0
  129. package/src/services/search.service.ts +761 -0
  130. package/src/services/search.types.ts +220 -0
  131. package/src/services/secrets.service.ts +384 -0
  132. package/src/services/vectorstore/base.ts +327 -0
  133. package/src/services/vectorstore/index.ts +444 -0
  134. package/src/services/vectorstore/memory.ts +286 -0
  135. package/src/services/vectorstore/migration.ts +295 -0
  136. package/src/services/vectorstore/mock.ts +403 -0
  137. package/src/services/vectorstore/pgvector.ts +695 -0
  138. package/src/services/vectorstore/types.ts +247 -0
  139. package/src/startup.ts +389 -0
  140. package/src/types/api.types.ts +193 -0
  141. package/src/types/document.types.ts +103 -0
  142. package/src/types/index.ts +241 -0
  143. package/src/types/profile.base.ts +133 -0
  144. package/src/utils/errors.ts +447 -0
  145. package/src/utils/id.ts +15 -0
  146. package/src/utils/index.ts +101 -0
  147. package/src/utils/logger.ts +313 -0
  148. package/src/utils/sanitization.ts +501 -0
  149. package/src/utils/secret-validation.ts +273 -0
  150. package/src/utils/synonyms.ts +188 -0
  151. package/src/utils/validation.ts +581 -0
  152. package/src/workers/chunking.worker.ts +242 -0
  153. package/src/workers/embedding.worker.ts +358 -0
  154. package/src/workers/extraction.worker.ts +346 -0
  155. package/src/workers/indexing.worker.ts +505 -0
  156. package/tsconfig.json +38 -0
@@ -0,0 +1,47 @@
1
+ /**
2
+ * Shared Database Connection for Workers
3
+ *
4
+ * Provides a singleton database connection pool and Drizzle instance
5
+ * for use across all BullMQ workers (extraction, chunking, embedding, indexing).
6
+ *
7
+ * Benefits:
8
+ * - DRY: Single source of truth for worker database configuration
9
+ * - Consistency: All workers use the same connection settings
10
+ * - Maintainability: Easy to update connection logic in one place
11
+ */
12
+
13
+ import { drizzle } from 'drizzle-orm/node-postgres'
14
+ import { Pool } from 'pg'
15
+ import * as schema from './schema/index.js'
16
+
17
+ /**
18
+ * Database connection URL with default fallback
19
+ */
20
+ export const DATABASE_URL =
21
+ process.env.DATABASE_URL || 'postgresql://supermemory:supermemory_secret@localhost:5432/supermemory'
22
+
23
+ /**
24
+ * Shared PostgreSQL connection pool
25
+ * Reused across all workers to prevent connection exhaustion
26
+ */
27
+ export const workerPool = new Pool({ connectionString: DATABASE_URL })
28
+
29
+ /**
30
+ * Shared Drizzle database instance with schema
31
+ * Provides type-safe database operations for all workers
32
+ */
33
+ export const workerDb = drizzle(workerPool, { schema })
34
+
35
+ /**
36
+ * Extract transaction type from database instance for type safety
37
+ * Use this type for transaction parameters in worker methods
38
+ */
39
+ export type WorkerTransaction = Parameters<Parameters<typeof workerDb.transaction>[0]>[0]
40
+
41
+ /**
42
+ * Gracefully close the worker database connection pool
43
+ * Call this during worker shutdown to prevent hanging connections
44
+ */
45
+ export async function closeWorkerDbConnection(): Promise<void> {
46
+ await workerPool.end()
47
+ }
package/src/index.ts ADDED
@@ -0,0 +1,235 @@
1
+ import './config/bootstrap-env.js'
2
+ import { serve } from '@hono/node-server'
3
+ import { Hono } from 'hono'
4
+ import { cors } from 'hono/cors'
5
+ import { logger } from 'hono/logger'
6
+ import { prettyJSON } from 'hono/pretty-json'
7
+ import { timing } from 'hono/timing'
8
+ import pkg from 'pg'
9
+ import { config } from './config/index.js'
10
+ import { getDatabase } from './db/client.js'
11
+
12
+ // Import route modules
13
+ import { documentsRouter } from './api/routes/documents.js'
14
+ import { searchRouter } from './api/routes/search.js'
15
+ import { profilesRouter } from './api/routes/profiles.js'
16
+ import { authMiddleware } from './api/middleware/auth.js'
17
+ import { errorHandlerMiddleware } from './api/middleware/errorHandler.js'
18
+ import { standardRateLimit } from './api/middleware/rateLimit.js'
19
+ import { setCsrfCookie, csrfProtection } from './api/middleware/csrf.js'
20
+ import { initializeAndValidate } from './startup.js'
21
+
22
+ const { Pool } = pkg
23
+
24
+ // Initialize database
25
+ const db = getDatabase()
26
+
27
+ let healthPool: pkg.Pool | null = null
28
+
29
+ function getHealthPool(): pkg.Pool {
30
+ if (!healthPool) {
31
+ healthPool = new Pool({
32
+ connectionString: config.databaseUrl,
33
+ max: 1,
34
+ idleTimeoutMillis: 30000,
35
+ connectionTimeoutMillis: 2000,
36
+ })
37
+ }
38
+
39
+ return healthPool
40
+ }
41
+
42
+ // Create Hono app
43
+ const app = new Hono()
44
+
45
+ // ============================================================================
46
+ // Global Middleware
47
+ // ============================================================================
48
+
49
+ // Error handler (must be first to catch all errors)
50
+ app.use('*', errorHandlerMiddleware)
51
+
52
+ // Request logging
53
+ app.use('*', logger())
54
+
55
+ // CORS configuration
56
+ app.use(
57
+ '*',
58
+ cors({
59
+ origin: ['http://localhost:3000', 'http://localhost:5173'],
60
+ allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
61
+ allowHeaders: ['Content-Type', 'Authorization', 'X-Request-ID', 'X-CSRF-Token'],
62
+ exposeHeaders: ['X-RateLimit-Limit', 'X-RateLimit-Remaining', 'X-RateLimit-Reset'],
63
+ credentials: true,
64
+ maxAge: 86400,
65
+ })
66
+ )
67
+
68
+ // Pretty JSON responses in development
69
+ if (process.env.NODE_ENV !== 'production') {
70
+ app.use('*', prettyJSON())
71
+ }
72
+
73
+ // Request timing
74
+ app.use('*', timing())
75
+
76
+ // ============================================================================
77
+ // Health Check (No Auth Required)
78
+ // ============================================================================
79
+
80
+ app.get('/health', async (c) => {
81
+ const checks = {
82
+ status: 'healthy',
83
+ version: '1.0.0',
84
+ timestamp: new Date().toISOString(),
85
+ }
86
+
87
+ try {
88
+ const pool = getHealthPool()
89
+ await pool.query('SELECT 1')
90
+ } catch {
91
+ checks.status = 'unhealthy'
92
+ }
93
+
94
+ const statusCode = checks.status === 'healthy' ? 200 : 503
95
+ return c.json(checks, statusCode)
96
+ })
97
+
98
+ app.get('/', (c) => {
99
+ return c.json({
100
+ name: 'Supermemory Clone API',
101
+ version: '1.0.0',
102
+ description: 'Personal AI memory assistant with semantic search',
103
+ endpoints: {
104
+ documents: '/api/v1/documents',
105
+ search: '/api/v1/search',
106
+ profiles: '/api/v1/profiles',
107
+ },
108
+ })
109
+ })
110
+
111
+ // CSRF token endpoint for SPA clients
112
+ // Apply CSRF cookie middleware to provide token
113
+ app.get('/api/v1/csrf-token', setCsrfCookie(), (c) => {
114
+ const csrfToken = c.get('csrfToken')
115
+
116
+ return c.json({
117
+ csrfToken,
118
+ expiresIn: 3600, // 1 hour in seconds
119
+ })
120
+ })
121
+
122
+ // ============================================================================
123
+ // API Routes (Auth Required)
124
+ // ============================================================================
125
+
126
+ const api = new Hono()
127
+
128
+ // Apply middleware stack to all API routes
129
+ // Order: auth → CSRF → rate limit → routes
130
+ api.use('*', authMiddleware)
131
+ api.use('*', setCsrfCookie())
132
+ api.use(
133
+ '*',
134
+ csrfProtection({
135
+ allowedOrigins: process.env.ALLOWED_ORIGINS?.split(',') || ['http://localhost:3000', 'http://localhost:5173'],
136
+ })
137
+ )
138
+ api.use('*', standardRateLimit)
139
+
140
+ // Mount route modules
141
+ api.route('/documents', documentsRouter)
142
+ api.route('/search', searchRouter)
143
+ api.route('/profiles', profilesRouter)
144
+
145
+ // Mount API under versioned path
146
+ app.route('/api/v1', api)
147
+
148
+ // ============================================================================
149
+ // 404 Handler
150
+ // ============================================================================
151
+
152
+ app.notFound((c) => {
153
+ return c.json(
154
+ {
155
+ error: {
156
+ code: 'NOT_FOUND',
157
+ message: `Route ${c.req.method} ${c.req.path} not found`,
158
+ },
159
+ status: 404,
160
+ },
161
+ 404
162
+ )
163
+ })
164
+
165
+ // ============================================================================
166
+ // Error Handler
167
+ // ============================================================================
168
+
169
+ app.onError((err, c) => {
170
+ console.error('Unhandled error:', err)
171
+ return c.json(
172
+ {
173
+ error: {
174
+ code: 'INTERNAL_SERVER_ERROR',
175
+ message: process.env.NODE_ENV === 'development' ? err.message : 'An unexpected error occurred',
176
+ },
177
+ status: 500,
178
+ },
179
+ 500
180
+ )
181
+ })
182
+
183
+ // ============================================================================
184
+ // Start Server
185
+ // ============================================================================
186
+
187
+ const port = config.apiPort
188
+ const host = config.apiHost
189
+
190
+ console.log(`
191
+ ================================================
192
+ Supermemory Clone API Server
193
+ ================================================
194
+ Version: 1.0.0
195
+ Port: ${port}
196
+ Host: ${host}
197
+ Environment: ${process.env.NODE_ENV ?? 'development'}
198
+ ================================================
199
+
200
+ Available endpoints:
201
+ GET / - API info
202
+ GET /health - Health check
203
+
204
+ POST /api/v1/documents - Create document
205
+ GET /api/v1/documents - List documents
206
+ GET /api/v1/documents/:id - Get document
207
+ PUT /api/v1/documents/:id - Update document
208
+ DELETE /api/v1/documents/:id - Delete document
209
+ POST /api/v1/documents/file - Upload file
210
+ POST /api/v1/documents/bulk-delete - Bulk delete
211
+
212
+ POST /api/v1/search - Search documents
213
+
214
+ GET /api/v1/profiles - List profiles
215
+ GET /api/v1/profiles/:tag - Get profile
216
+ PUT /api/v1/profiles/:tag - Update profile
217
+ DELETE /api/v1/profiles/:tag - Delete profile
218
+ `)
219
+
220
+ async function startServer(): Promise<void> {
221
+ await initializeAndValidate()
222
+
223
+ serve({
224
+ fetch: app.fetch,
225
+ port,
226
+ hostname: host,
227
+ })
228
+ }
229
+
230
+ startServer().catch((error) => {
231
+ console.error('Failed to start API server:', error)
232
+ process.exit(1)
233
+ })
234
+
235
+ export { app, db }
@@ -0,0 +1 @@
1
+