@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Shared Database Connection for Workers
|
|
3
|
+
*
|
|
4
|
+
* Provides a singleton database connection pool and Drizzle instance
|
|
5
|
+
* for use across all BullMQ workers (extraction, chunking, embedding, indexing).
|
|
6
|
+
*
|
|
7
|
+
* Benefits:
|
|
8
|
+
* - DRY: Single source of truth for worker database configuration
|
|
9
|
+
* - Consistency: All workers use the same connection settings
|
|
10
|
+
* - Maintainability: Easy to update connection logic in one place
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { drizzle } from 'drizzle-orm/node-postgres'
|
|
14
|
+
import { Pool } from 'pg'
|
|
15
|
+
import * as schema from './schema/index.js'
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Database connection URL with default fallback
|
|
19
|
+
*/
|
|
20
|
+
export const DATABASE_URL =
|
|
21
|
+
process.env.DATABASE_URL || 'postgresql://supermemory:supermemory_secret@localhost:5432/supermemory'
|
|
22
|
+
|
|
23
|
+
/**
|
|
24
|
+
* Shared PostgreSQL connection pool
|
|
25
|
+
* Reused across all workers to prevent connection exhaustion
|
|
26
|
+
*/
|
|
27
|
+
export const workerPool = new Pool({ connectionString: DATABASE_URL })
|
|
28
|
+
|
|
29
|
+
/**
|
|
30
|
+
* Shared Drizzle database instance with schema
|
|
31
|
+
* Provides type-safe database operations for all workers
|
|
32
|
+
*/
|
|
33
|
+
export const workerDb = drizzle(workerPool, { schema })
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Extract transaction type from database instance for type safety
|
|
37
|
+
* Use this type for transaction parameters in worker methods
|
|
38
|
+
*/
|
|
39
|
+
export type WorkerTransaction = Parameters<Parameters<typeof workerDb.transaction>[0]>[0]
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* Gracefully close the worker database connection pool
|
|
43
|
+
* Call this during worker shutdown to prevent hanging connections
|
|
44
|
+
*/
|
|
45
|
+
export async function closeWorkerDbConnection(): Promise<void> {
|
|
46
|
+
await workerPool.end()
|
|
47
|
+
}
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,235 @@
|
|
|
1
|
+
import './config/bootstrap-env.js'
|
|
2
|
+
import { serve } from '@hono/node-server'
|
|
3
|
+
import { Hono } from 'hono'
|
|
4
|
+
import { cors } from 'hono/cors'
|
|
5
|
+
import { logger } from 'hono/logger'
|
|
6
|
+
import { prettyJSON } from 'hono/pretty-json'
|
|
7
|
+
import { timing } from 'hono/timing'
|
|
8
|
+
import pkg from 'pg'
|
|
9
|
+
import { config } from './config/index.js'
|
|
10
|
+
import { getDatabase } from './db/client.js'
|
|
11
|
+
|
|
12
|
+
// Import route modules
|
|
13
|
+
import { documentsRouter } from './api/routes/documents.js'
|
|
14
|
+
import { searchRouter } from './api/routes/search.js'
|
|
15
|
+
import { profilesRouter } from './api/routes/profiles.js'
|
|
16
|
+
import { authMiddleware } from './api/middleware/auth.js'
|
|
17
|
+
import { errorHandlerMiddleware } from './api/middleware/errorHandler.js'
|
|
18
|
+
import { standardRateLimit } from './api/middleware/rateLimit.js'
|
|
19
|
+
import { setCsrfCookie, csrfProtection } from './api/middleware/csrf.js'
|
|
20
|
+
import { initializeAndValidate } from './startup.js'
|
|
21
|
+
|
|
22
|
+
const { Pool } = pkg
|
|
23
|
+
|
|
24
|
+
// Initialize database
|
|
25
|
+
const db = getDatabase()
|
|
26
|
+
|
|
27
|
+
let healthPool: pkg.Pool | null = null
|
|
28
|
+
|
|
29
|
+
function getHealthPool(): pkg.Pool {
|
|
30
|
+
if (!healthPool) {
|
|
31
|
+
healthPool = new Pool({
|
|
32
|
+
connectionString: config.databaseUrl,
|
|
33
|
+
max: 1,
|
|
34
|
+
idleTimeoutMillis: 30000,
|
|
35
|
+
connectionTimeoutMillis: 2000,
|
|
36
|
+
})
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
return healthPool
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
// Create Hono app
|
|
43
|
+
const app = new Hono()
|
|
44
|
+
|
|
45
|
+
// ============================================================================
|
|
46
|
+
// Global Middleware
|
|
47
|
+
// ============================================================================
|
|
48
|
+
|
|
49
|
+
// Error handler (must be first to catch all errors)
|
|
50
|
+
app.use('*', errorHandlerMiddleware)
|
|
51
|
+
|
|
52
|
+
// Request logging
|
|
53
|
+
app.use('*', logger())
|
|
54
|
+
|
|
55
|
+
// CORS configuration
|
|
56
|
+
app.use(
|
|
57
|
+
'*',
|
|
58
|
+
cors({
|
|
59
|
+
origin: ['http://localhost:3000', 'http://localhost:5173'],
|
|
60
|
+
allowMethods: ['GET', 'POST', 'PUT', 'DELETE', 'OPTIONS'],
|
|
61
|
+
allowHeaders: ['Content-Type', 'Authorization', 'X-Request-ID', 'X-CSRF-Token'],
|
|
62
|
+
exposeHeaders: ['X-RateLimit-Limit', 'X-RateLimit-Remaining', 'X-RateLimit-Reset'],
|
|
63
|
+
credentials: true,
|
|
64
|
+
maxAge: 86400,
|
|
65
|
+
})
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
// Pretty JSON responses in development
|
|
69
|
+
if (process.env.NODE_ENV !== 'production') {
|
|
70
|
+
app.use('*', prettyJSON())
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Request timing
|
|
74
|
+
app.use('*', timing())
|
|
75
|
+
|
|
76
|
+
// ============================================================================
|
|
77
|
+
// Health Check (No Auth Required)
|
|
78
|
+
// ============================================================================
|
|
79
|
+
|
|
80
|
+
app.get('/health', async (c) => {
|
|
81
|
+
const checks = {
|
|
82
|
+
status: 'healthy',
|
|
83
|
+
version: '1.0.0',
|
|
84
|
+
timestamp: new Date().toISOString(),
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
try {
|
|
88
|
+
const pool = getHealthPool()
|
|
89
|
+
await pool.query('SELECT 1')
|
|
90
|
+
} catch {
|
|
91
|
+
checks.status = 'unhealthy'
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
const statusCode = checks.status === 'healthy' ? 200 : 503
|
|
95
|
+
return c.json(checks, statusCode)
|
|
96
|
+
})
|
|
97
|
+
|
|
98
|
+
app.get('/', (c) => {
|
|
99
|
+
return c.json({
|
|
100
|
+
name: 'Supermemory Clone API',
|
|
101
|
+
version: '1.0.0',
|
|
102
|
+
description: 'Personal AI memory assistant with semantic search',
|
|
103
|
+
endpoints: {
|
|
104
|
+
documents: '/api/v1/documents',
|
|
105
|
+
search: '/api/v1/search',
|
|
106
|
+
profiles: '/api/v1/profiles',
|
|
107
|
+
},
|
|
108
|
+
})
|
|
109
|
+
})
|
|
110
|
+
|
|
111
|
+
// CSRF token endpoint for SPA clients
|
|
112
|
+
// Apply CSRF cookie middleware to provide token
|
|
113
|
+
app.get('/api/v1/csrf-token', setCsrfCookie(), (c) => {
|
|
114
|
+
const csrfToken = c.get('csrfToken')
|
|
115
|
+
|
|
116
|
+
return c.json({
|
|
117
|
+
csrfToken,
|
|
118
|
+
expiresIn: 3600, // 1 hour in seconds
|
|
119
|
+
})
|
|
120
|
+
})
|
|
121
|
+
|
|
122
|
+
// ============================================================================
|
|
123
|
+
// API Routes (Auth Required)
|
|
124
|
+
// ============================================================================
|
|
125
|
+
|
|
126
|
+
const api = new Hono()
|
|
127
|
+
|
|
128
|
+
// Apply middleware stack to all API routes
|
|
129
|
+
// Order: auth → CSRF → rate limit → routes
|
|
130
|
+
api.use('*', authMiddleware)
|
|
131
|
+
api.use('*', setCsrfCookie())
|
|
132
|
+
api.use(
|
|
133
|
+
'*',
|
|
134
|
+
csrfProtection({
|
|
135
|
+
allowedOrigins: process.env.ALLOWED_ORIGINS?.split(',') || ['http://localhost:3000', 'http://localhost:5173'],
|
|
136
|
+
})
|
|
137
|
+
)
|
|
138
|
+
api.use('*', standardRateLimit)
|
|
139
|
+
|
|
140
|
+
// Mount route modules
|
|
141
|
+
api.route('/documents', documentsRouter)
|
|
142
|
+
api.route('/search', searchRouter)
|
|
143
|
+
api.route('/profiles', profilesRouter)
|
|
144
|
+
|
|
145
|
+
// Mount API under versioned path
|
|
146
|
+
app.route('/api/v1', api)
|
|
147
|
+
|
|
148
|
+
// ============================================================================
|
|
149
|
+
// 404 Handler
|
|
150
|
+
// ============================================================================
|
|
151
|
+
|
|
152
|
+
app.notFound((c) => {
|
|
153
|
+
return c.json(
|
|
154
|
+
{
|
|
155
|
+
error: {
|
|
156
|
+
code: 'NOT_FOUND',
|
|
157
|
+
message: `Route ${c.req.method} ${c.req.path} not found`,
|
|
158
|
+
},
|
|
159
|
+
status: 404,
|
|
160
|
+
},
|
|
161
|
+
404
|
|
162
|
+
)
|
|
163
|
+
})
|
|
164
|
+
|
|
165
|
+
// ============================================================================
|
|
166
|
+
// Error Handler
|
|
167
|
+
// ============================================================================
|
|
168
|
+
|
|
169
|
+
app.onError((err, c) => {
|
|
170
|
+
console.error('Unhandled error:', err)
|
|
171
|
+
return c.json(
|
|
172
|
+
{
|
|
173
|
+
error: {
|
|
174
|
+
code: 'INTERNAL_SERVER_ERROR',
|
|
175
|
+
message: process.env.NODE_ENV === 'development' ? err.message : 'An unexpected error occurred',
|
|
176
|
+
},
|
|
177
|
+
status: 500,
|
|
178
|
+
},
|
|
179
|
+
500
|
|
180
|
+
)
|
|
181
|
+
})
|
|
182
|
+
|
|
183
|
+
// ============================================================================
|
|
184
|
+
// Start Server
|
|
185
|
+
// ============================================================================
|
|
186
|
+
|
|
187
|
+
const port = config.apiPort
|
|
188
|
+
const host = config.apiHost
|
|
189
|
+
|
|
190
|
+
console.log(`
|
|
191
|
+
================================================
|
|
192
|
+
Supermemory Clone API Server
|
|
193
|
+
================================================
|
|
194
|
+
Version: 1.0.0
|
|
195
|
+
Port: ${port}
|
|
196
|
+
Host: ${host}
|
|
197
|
+
Environment: ${process.env.NODE_ENV ?? 'development'}
|
|
198
|
+
================================================
|
|
199
|
+
|
|
200
|
+
Available endpoints:
|
|
201
|
+
GET / - API info
|
|
202
|
+
GET /health - Health check
|
|
203
|
+
|
|
204
|
+
POST /api/v1/documents - Create document
|
|
205
|
+
GET /api/v1/documents - List documents
|
|
206
|
+
GET /api/v1/documents/:id - Get document
|
|
207
|
+
PUT /api/v1/documents/:id - Update document
|
|
208
|
+
DELETE /api/v1/documents/:id - Delete document
|
|
209
|
+
POST /api/v1/documents/file - Upload file
|
|
210
|
+
POST /api/v1/documents/bulk-delete - Bulk delete
|
|
211
|
+
|
|
212
|
+
POST /api/v1/search - Search documents
|
|
213
|
+
|
|
214
|
+
GET /api/v1/profiles - List profiles
|
|
215
|
+
GET /api/v1/profiles/:tag - Get profile
|
|
216
|
+
PUT /api/v1/profiles/:tag - Update profile
|
|
217
|
+
DELETE /api/v1/profiles/:tag - Delete profile
|
|
218
|
+
`)
|
|
219
|
+
|
|
220
|
+
async function startServer(): Promise<void> {
|
|
221
|
+
await initializeAndValidate()
|
|
222
|
+
|
|
223
|
+
serve({
|
|
224
|
+
fetch: app.fetch,
|
|
225
|
+
port,
|
|
226
|
+
hostname: host,
|
|
227
|
+
})
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
startServer().catch((error) => {
|
|
231
|
+
console.error('Failed to start API server:', error)
|
|
232
|
+
process.exit(1)
|
|
233
|
+
})
|
|
234
|
+
|
|
235
|
+
export { app, db }
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
|