@twelvehart/supermemory-runtime 1.0.0-next.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.env.example +57 -0
- package/README.md +374 -0
- package/dist/index.js +189 -0
- package/dist/mcp/index.js +1132 -0
- package/docker-compose.prod.yml +91 -0
- package/docker-compose.yml +358 -0
- package/drizzle/0000_dapper_the_professor.sql +159 -0
- package/drizzle/0001_api_keys.sql +51 -0
- package/drizzle/meta/0000_snapshot.json +1532 -0
- package/drizzle/meta/_journal.json +13 -0
- package/drizzle.config.ts +20 -0
- package/package.json +114 -0
- package/scripts/add-extraction-job.ts +122 -0
- package/scripts/benchmark-pgvector.ts +122 -0
- package/scripts/bootstrap.sh +209 -0
- package/scripts/check-runtime-pack.ts +111 -0
- package/scripts/claude-mcp-config.ts +336 -0
- package/scripts/docker-entrypoint.sh +183 -0
- package/scripts/doctor.ts +377 -0
- package/scripts/init-db.sql +33 -0
- package/scripts/install.sh +1110 -0
- package/scripts/mcp-setup.ts +271 -0
- package/scripts/migrations/001_create_pgvector_extension.sql +31 -0
- package/scripts/migrations/002_create_memory_embeddings_table.sql +75 -0
- package/scripts/migrations/003_create_hnsw_index.sql +94 -0
- package/scripts/migrations/004_create_memory_embeddings_standalone.sql +70 -0
- package/scripts/migrations/005_create_chunks_table.sql +95 -0
- package/scripts/migrations/006_create_processing_queue.sql +45 -0
- package/scripts/migrations/generate_test_data.sql +42 -0
- package/scripts/migrations/phase1_comprehensive_test.sql +204 -0
- package/scripts/migrations/run_migrations.sh +286 -0
- package/scripts/migrations/test_hnsw_index.sql +255 -0
- package/scripts/pre-commit-secrets +282 -0
- package/scripts/run-extraction-worker.ts +46 -0
- package/scripts/run-phase1-tests.sh +291 -0
- package/scripts/setup.ts +222 -0
- package/scripts/smoke-install.sh +12 -0
- package/scripts/test-health-endpoint.sh +328 -0
- package/src/api/index.ts +2 -0
- package/src/api/middleware/auth.ts +80 -0
- package/src/api/middleware/csrf.ts +308 -0
- package/src/api/middleware/errorHandler.ts +166 -0
- package/src/api/middleware/rateLimit.ts +360 -0
- package/src/api/middleware/validation.ts +514 -0
- package/src/api/routes/documents.ts +286 -0
- package/src/api/routes/profiles.ts +237 -0
- package/src/api/routes/search.ts +71 -0
- package/src/api/stores/index.ts +58 -0
- package/src/config/bootstrap-env.ts +3 -0
- package/src/config/env.ts +71 -0
- package/src/config/feature-flags.ts +25 -0
- package/src/config/index.ts +140 -0
- package/src/config/secrets.config.ts +291 -0
- package/src/db/client.ts +92 -0
- package/src/db/index.ts +73 -0
- package/src/db/postgres.ts +72 -0
- package/src/db/schema/chunks.schema.ts +31 -0
- package/src/db/schema/containers.schema.ts +46 -0
- package/src/db/schema/documents.schema.ts +49 -0
- package/src/db/schema/embeddings.schema.ts +32 -0
- package/src/db/schema/index.ts +11 -0
- package/src/db/schema/memories.schema.ts +72 -0
- package/src/db/schema/profiles.schema.ts +34 -0
- package/src/db/schema/queue.schema.ts +59 -0
- package/src/db/schema/relationships.schema.ts +42 -0
- package/src/db/schema.ts +223 -0
- package/src/db/worker-connection.ts +47 -0
- package/src/index.ts +235 -0
- package/src/mcp/CLAUDE.md +1 -0
- package/src/mcp/index.ts +1380 -0
- package/src/mcp/legacyState.ts +22 -0
- package/src/mcp/rateLimit.ts +358 -0
- package/src/mcp/resources.ts +309 -0
- package/src/mcp/results.ts +104 -0
- package/src/mcp/tools.ts +401 -0
- package/src/queues/config.ts +119 -0
- package/src/queues/index.ts +289 -0
- package/src/sdk/client.ts +225 -0
- package/src/sdk/errors.ts +266 -0
- package/src/sdk/http.ts +560 -0
- package/src/sdk/index.ts +244 -0
- package/src/sdk/resources/base.ts +65 -0
- package/src/sdk/resources/connections.ts +204 -0
- package/src/sdk/resources/documents.ts +163 -0
- package/src/sdk/resources/index.ts +10 -0
- package/src/sdk/resources/memories.ts +150 -0
- package/src/sdk/resources/search.ts +60 -0
- package/src/sdk/resources/settings.ts +36 -0
- package/src/sdk/types.ts +674 -0
- package/src/services/chunking/index.ts +451 -0
- package/src/services/chunking.service.ts +650 -0
- package/src/services/csrf.service.ts +252 -0
- package/src/services/documents.repository.ts +219 -0
- package/src/services/documents.service.ts +191 -0
- package/src/services/embedding.service.ts +404 -0
- package/src/services/extraction.service.ts +300 -0
- package/src/services/extractors/code.extractor.ts +451 -0
- package/src/services/extractors/index.ts +9 -0
- package/src/services/extractors/markdown.extractor.ts +461 -0
- package/src/services/extractors/pdf.extractor.ts +315 -0
- package/src/services/extractors/text.extractor.ts +118 -0
- package/src/services/extractors/url.extractor.ts +243 -0
- package/src/services/index.ts +235 -0
- package/src/services/ingestion.service.ts +177 -0
- package/src/services/llm/anthropic.ts +400 -0
- package/src/services/llm/base.ts +460 -0
- package/src/services/llm/contradiction-detector.service.ts +526 -0
- package/src/services/llm/heuristics.ts +148 -0
- package/src/services/llm/index.ts +309 -0
- package/src/services/llm/memory-classifier.service.ts +383 -0
- package/src/services/llm/memory-extension-detector.service.ts +523 -0
- package/src/services/llm/mock.ts +470 -0
- package/src/services/llm/openai.ts +398 -0
- package/src/services/llm/prompts.ts +438 -0
- package/src/services/llm/types.ts +373 -0
- package/src/services/memory.repository.ts +1769 -0
- package/src/services/memory.service.ts +1338 -0
- package/src/services/memory.types.ts +234 -0
- package/src/services/persistence/index.ts +295 -0
- package/src/services/pipeline.service.ts +509 -0
- package/src/services/profile.repository.ts +436 -0
- package/src/services/profile.service.ts +560 -0
- package/src/services/profile.types.ts +270 -0
- package/src/services/relationships/detector.ts +1128 -0
- package/src/services/relationships/index.ts +268 -0
- package/src/services/relationships/memory-integration.ts +459 -0
- package/src/services/relationships/strategies.ts +132 -0
- package/src/services/relationships/types.ts +370 -0
- package/src/services/search.service.ts +761 -0
- package/src/services/search.types.ts +220 -0
- package/src/services/secrets.service.ts +384 -0
- package/src/services/vectorstore/base.ts +327 -0
- package/src/services/vectorstore/index.ts +444 -0
- package/src/services/vectorstore/memory.ts +286 -0
- package/src/services/vectorstore/migration.ts +295 -0
- package/src/services/vectorstore/mock.ts +403 -0
- package/src/services/vectorstore/pgvector.ts +695 -0
- package/src/services/vectorstore/types.ts +247 -0
- package/src/startup.ts +389 -0
- package/src/types/api.types.ts +193 -0
- package/src/types/document.types.ts +103 -0
- package/src/types/index.ts +241 -0
- package/src/types/profile.base.ts +133 -0
- package/src/utils/errors.ts +447 -0
- package/src/utils/id.ts +15 -0
- package/src/utils/index.ts +101 -0
- package/src/utils/logger.ts +313 -0
- package/src/utils/sanitization.ts +501 -0
- package/src/utils/secret-validation.ts +273 -0
- package/src/utils/synonyms.ts +188 -0
- package/src/utils/validation.ts +581 -0
- package/src/workers/chunking.worker.ts +242 -0
- package/src/workers/embedding.worker.ts +358 -0
- package/src/workers/extraction.worker.ts +346 -0
- package/src/workers/indexing.worker.ts +505 -0
- package/tsconfig.json +38 -0
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
import { Context, MiddlewareHandler } from 'hono'
|
|
2
|
+
import { HTTPException } from 'hono/http-exception'
|
|
3
|
+
import { ZodError } from 'zod'
|
|
4
|
+
import { ErrorCodes, ErrorResponse } from '../../types/api.types.js'
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Custom API error class for consistent error handling.
|
|
8
|
+
*/
|
|
9
|
+
export class ApiError extends Error {
|
|
10
|
+
constructor(
|
|
11
|
+
public readonly code: string,
|
|
12
|
+
message: string,
|
|
13
|
+
public readonly statusCode: number = 400,
|
|
14
|
+
public readonly details?: Record<string, unknown>
|
|
15
|
+
) {
|
|
16
|
+
super(message)
|
|
17
|
+
this.name = 'ApiError'
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
toResponse(): ErrorResponse {
|
|
21
|
+
return {
|
|
22
|
+
error: {
|
|
23
|
+
code: this.code,
|
|
24
|
+
message: this.message,
|
|
25
|
+
...(this.details && { details: this.details }),
|
|
26
|
+
},
|
|
27
|
+
status: this.statusCode,
|
|
28
|
+
} as ErrorResponse
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/**
|
|
33
|
+
* Formats Zod validation errors into a readable format.
|
|
34
|
+
*/
|
|
35
|
+
function formatZodErrors(error: ZodError): string {
|
|
36
|
+
const issues = error.issues.map((issue) => {
|
|
37
|
+
const path = issue.path.join('.')
|
|
38
|
+
return path ? `${path}: ${issue.message}` : issue.message
|
|
39
|
+
})
|
|
40
|
+
return issues.join('; ')
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Global error handler middleware.
|
|
45
|
+
* Catches all errors and returns consistent error responses.
|
|
46
|
+
*/
|
|
47
|
+
export const errorHandlerMiddleware: MiddlewareHandler = async (c: Context, next) => {
|
|
48
|
+
try {
|
|
49
|
+
return await next()
|
|
50
|
+
} catch (error) {
|
|
51
|
+
console.error('Error caught in error handler:', error)
|
|
52
|
+
|
|
53
|
+
// Handle custom API errors
|
|
54
|
+
if (error instanceof ApiError) {
|
|
55
|
+
const statusCode = error.statusCode as 400 | 401 | 403 | 404 | 409 | 429 | 500
|
|
56
|
+
return c.json(error.toResponse(), statusCode)
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// Handle Zod validation errors
|
|
60
|
+
if (error instanceof ZodError) {
|
|
61
|
+
const response: ErrorResponse = {
|
|
62
|
+
error: {
|
|
63
|
+
code: ErrorCodes.VALIDATION_ERROR,
|
|
64
|
+
message: formatZodErrors(error),
|
|
65
|
+
},
|
|
66
|
+
status: 400,
|
|
67
|
+
}
|
|
68
|
+
return c.json(response, 400)
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// Handle Hono HTTP exceptions
|
|
72
|
+
if (error instanceof HTTPException) {
|
|
73
|
+
const response: ErrorResponse = {
|
|
74
|
+
error: {
|
|
75
|
+
code: mapHttpStatusToCode(error.status),
|
|
76
|
+
message: error.message || getDefaultMessage(error.status),
|
|
77
|
+
},
|
|
78
|
+
status: error.status,
|
|
79
|
+
}
|
|
80
|
+
return c.json(response, error.status)
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
// Handle generic errors
|
|
84
|
+
const message = error instanceof Error ? error.message : 'An unexpected error occurred'
|
|
85
|
+
const response: ErrorResponse = {
|
|
86
|
+
error: {
|
|
87
|
+
code: ErrorCodes.INTERNAL_ERROR,
|
|
88
|
+
message: process.env.NODE_ENV === 'production' ? 'An unexpected error occurred' : message,
|
|
89
|
+
},
|
|
90
|
+
status: 500,
|
|
91
|
+
}
|
|
92
|
+
return c.json(response, 500)
|
|
93
|
+
}
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Maps HTTP status codes to error codes.
|
|
98
|
+
*/
|
|
99
|
+
function mapHttpStatusToCode(status: number): string {
|
|
100
|
+
switch (status) {
|
|
101
|
+
case 400:
|
|
102
|
+
return ErrorCodes.BAD_REQUEST
|
|
103
|
+
case 401:
|
|
104
|
+
return ErrorCodes.UNAUTHORIZED
|
|
105
|
+
case 403:
|
|
106
|
+
return ErrorCodes.FORBIDDEN
|
|
107
|
+
case 404:
|
|
108
|
+
return ErrorCodes.NOT_FOUND
|
|
109
|
+
case 409:
|
|
110
|
+
return ErrorCodes.CONFLICT
|
|
111
|
+
case 429:
|
|
112
|
+
return ErrorCodes.RATE_LIMITED
|
|
113
|
+
default:
|
|
114
|
+
return ErrorCodes.INTERNAL_ERROR
|
|
115
|
+
}
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
/**
|
|
119
|
+
* Gets default error message for HTTP status codes.
|
|
120
|
+
*/
|
|
121
|
+
function getDefaultMessage(status: number): string {
|
|
122
|
+
switch (status) {
|
|
123
|
+
case 400:
|
|
124
|
+
return 'Bad request'
|
|
125
|
+
case 401:
|
|
126
|
+
return 'Unauthorized'
|
|
127
|
+
case 403:
|
|
128
|
+
return 'Forbidden'
|
|
129
|
+
case 404:
|
|
130
|
+
return 'Resource not found'
|
|
131
|
+
case 409:
|
|
132
|
+
return 'Resource conflict'
|
|
133
|
+
case 429:
|
|
134
|
+
return 'Too many requests'
|
|
135
|
+
default:
|
|
136
|
+
return 'Internal server error'
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/**
|
|
141
|
+
* Helper function to throw not found errors.
|
|
142
|
+
*/
|
|
143
|
+
export function notFound(resource: string, id: string): never {
|
|
144
|
+
throw new ApiError(ErrorCodes.NOT_FOUND, `${resource} with id '${id}' not found`, 404)
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* Helper function to throw validation errors.
|
|
149
|
+
*/
|
|
150
|
+
export function validationError(message: string): never {
|
|
151
|
+
throw new ApiError(ErrorCodes.VALIDATION_ERROR, message, 400)
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Helper function to throw conflict errors.
|
|
156
|
+
*/
|
|
157
|
+
export function conflict(message: string): never {
|
|
158
|
+
throw new ApiError(ErrorCodes.CONFLICT, message, 409)
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
/**
|
|
162
|
+
* Helper function to throw forbidden errors.
|
|
163
|
+
*/
|
|
164
|
+
export function forbidden(message: string): never {
|
|
165
|
+
throw new ApiError(ErrorCodes.FORBIDDEN, message, 403)
|
|
166
|
+
}
|
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
import { Context, MiddlewareHandler } from 'hono'
|
|
2
|
+
import { ErrorCodes } from '../../types/api.types.js'
|
|
3
|
+
import { getLogger } from '../../utils/logger.js'
|
|
4
|
+
|
|
5
|
+
const logger = getLogger('rate-limit')
|
|
6
|
+
|
|
7
|
+
interface RateLimitEntry {
|
|
8
|
+
count: number
|
|
9
|
+
resetTime: number
|
|
10
|
+
}
|
|
11
|
+
|
|
12
|
+
function isRateLimitEntry(value: unknown): value is RateLimitEntry {
|
|
13
|
+
if (!value || typeof value !== 'object') {
|
|
14
|
+
return false
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
const candidate = value as Partial<RateLimitEntry>
|
|
18
|
+
return (
|
|
19
|
+
typeof candidate.count === 'number' &&
|
|
20
|
+
Number.isFinite(candidate.count) &&
|
|
21
|
+
typeof candidate.resetTime === 'number' &&
|
|
22
|
+
Number.isFinite(candidate.resetTime)
|
|
23
|
+
)
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Rate limit store interface for pluggable backends
|
|
28
|
+
*/
|
|
29
|
+
export interface RateLimitStore {
|
|
30
|
+
get(key: string): Promise<RateLimitEntry | undefined>
|
|
31
|
+
set(key: string, entry: RateLimitEntry, ttlMs: number): Promise<void>
|
|
32
|
+
increment(key: string, windowMs: number): Promise<RateLimitEntry>
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* In-memory rate limit store for development/single-instance deployments
|
|
37
|
+
*/
|
|
38
|
+
export class MemoryRateLimitStore implements RateLimitStore {
|
|
39
|
+
private store = new Map<string, RateLimitEntry>()
|
|
40
|
+
private cleanupInterval: ReturnType<typeof setInterval> | null = null
|
|
41
|
+
|
|
42
|
+
constructor() {
|
|
43
|
+
// Cleanup old entries periodically
|
|
44
|
+
this.cleanupInterval = setInterval(() => {
|
|
45
|
+
const now = Date.now()
|
|
46
|
+
for (const [key, entry] of this.store.entries()) {
|
|
47
|
+
if (entry.resetTime <= now) {
|
|
48
|
+
this.store.delete(key)
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
}, 60 * 1000)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
async get(key: string): Promise<RateLimitEntry | undefined> {
|
|
55
|
+
const entry = this.store.get(key)
|
|
56
|
+
if (entry && entry.resetTime > Date.now()) {
|
|
57
|
+
return entry
|
|
58
|
+
}
|
|
59
|
+
return undefined
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async set(key: string, entry: RateLimitEntry, _ttlMs: number): Promise<void> {
|
|
63
|
+
this.store.set(key, entry)
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
async increment(key: string, windowMs: number): Promise<RateLimitEntry> {
|
|
67
|
+
const now = Date.now()
|
|
68
|
+
let entry = this.store.get(key)
|
|
69
|
+
|
|
70
|
+
if (!entry || entry.resetTime <= now) {
|
|
71
|
+
entry = {
|
|
72
|
+
count: 1,
|
|
73
|
+
resetTime: now + windowMs,
|
|
74
|
+
}
|
|
75
|
+
} else {
|
|
76
|
+
entry.count++
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
this.store.set(key, entry)
|
|
80
|
+
return entry
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
destroy(): void {
|
|
84
|
+
if (this.cleanupInterval) {
|
|
85
|
+
clearInterval(this.cleanupInterval)
|
|
86
|
+
this.cleanupInterval = null
|
|
87
|
+
}
|
|
88
|
+
this.store.clear()
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
/**
|
|
93
|
+
* Redis rate limit store for distributed deployments
|
|
94
|
+
*
|
|
95
|
+
* Requires REDIS_URL environment variable to be set.
|
|
96
|
+
* Falls back to memory store if Redis is not available.
|
|
97
|
+
*/
|
|
98
|
+
export class RedisRateLimitStore implements RateLimitStore {
|
|
99
|
+
private redis: RedisClient | null = null
|
|
100
|
+
private readonly keyPrefix = 'ratelimit:'
|
|
101
|
+
private fallback: MemoryRateLimitStore
|
|
102
|
+
private connectionFailed = false
|
|
103
|
+
|
|
104
|
+
constructor() {
|
|
105
|
+
this.fallback = new MemoryRateLimitStore()
|
|
106
|
+
this.initRedis()
|
|
107
|
+
}
|
|
108
|
+
|
|
109
|
+
private async initRedis(): Promise<void> {
|
|
110
|
+
const redisUrl = process.env.REDIS_URL
|
|
111
|
+
if (!redisUrl) {
|
|
112
|
+
logger.warn('REDIS_URL not set, using in-memory store')
|
|
113
|
+
return
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
try {
|
|
117
|
+
// Dynamic import to avoid requiring ioredis in all environments
|
|
118
|
+
// We catch the import error and fall back to in-memory store
|
|
119
|
+
const redisModule = await import('ioredis').catch(() => {
|
|
120
|
+
logger.warn('ioredis module not installed, using in-memory store')
|
|
121
|
+
return null
|
|
122
|
+
})
|
|
123
|
+
|
|
124
|
+
if (!redisModule) {
|
|
125
|
+
return
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// ioredis exports the class as default - use type assertion for dynamic import
|
|
129
|
+
const RedisConstructor = (redisModule.default || redisModule) as unknown as new (url: string) => RedisClient
|
|
130
|
+
const client = new RedisConstructor(redisUrl)
|
|
131
|
+
this.redis = client
|
|
132
|
+
|
|
133
|
+
client.on('error', (err: unknown) => {
|
|
134
|
+
const errMsg =
|
|
135
|
+
err && typeof err === 'object' && 'message' in err ? (err as { message: string }).message : 'Unknown error'
|
|
136
|
+
logger.error('Redis error', { error: errMsg })
|
|
137
|
+
this.connectionFailed = true
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
client.on('connect', () => {
|
|
141
|
+
logger.info('Connected to Redis')
|
|
142
|
+
this.connectionFailed = false
|
|
143
|
+
})
|
|
144
|
+
|
|
145
|
+
// ioredis connects automatically on construction
|
|
146
|
+
} catch (err) {
|
|
147
|
+
const message = err instanceof Error ? err.message : 'Unknown error'
|
|
148
|
+
logger.warn('Redis connection failed, using in-memory fallback', { error: message })
|
|
149
|
+
this.connectionFailed = true
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
|
|
153
|
+
private isAvailable(): boolean {
|
|
154
|
+
return this.redis !== null && !this.connectionFailed && this.redis.status === 'ready'
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
async get(key: string): Promise<RateLimitEntry | undefined> {
|
|
158
|
+
if (!this.isAvailable()) {
|
|
159
|
+
return this.fallback.get(key)
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
try {
|
|
163
|
+
const data = await this.redis!.get(this.keyPrefix + key)
|
|
164
|
+
if (data) {
|
|
165
|
+
const parsed = JSON.parse(data) as unknown
|
|
166
|
+
if (isRateLimitEntry(parsed)) {
|
|
167
|
+
return parsed
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
logger.warn('Redis returned malformed rate limit entry; falling back to memory store', { key })
|
|
171
|
+
}
|
|
172
|
+
return undefined
|
|
173
|
+
} catch (err) {
|
|
174
|
+
logger.error('Redis get error', { key }, err instanceof Error ? err : undefined)
|
|
175
|
+
return this.fallback.get(key)
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
async set(key: string, entry: RateLimitEntry, ttlMs: number): Promise<void> {
|
|
180
|
+
if (!this.isAvailable()) {
|
|
181
|
+
return this.fallback.set(key, entry, ttlMs)
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
try {
|
|
185
|
+
await this.redis!.set(this.keyPrefix + key, JSON.stringify(entry), 'PX', ttlMs)
|
|
186
|
+
} catch (err) {
|
|
187
|
+
logger.error('Redis set error', { key }, err instanceof Error ? err : undefined)
|
|
188
|
+
await this.fallback.set(key, entry, ttlMs)
|
|
189
|
+
}
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
async increment(key: string, windowMs: number): Promise<RateLimitEntry> {
|
|
193
|
+
if (!this.isAvailable()) {
|
|
194
|
+
return this.fallback.increment(key, windowMs)
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
try {
|
|
198
|
+
const now = Date.now()
|
|
199
|
+
const redisKey = this.keyPrefix + key
|
|
200
|
+
|
|
201
|
+
// Use Redis MULTI/EXEC for atomic increment with expiry
|
|
202
|
+
const result = await this.redis!.multi()
|
|
203
|
+
.incr(redisKey)
|
|
204
|
+
.pexpireat(redisKey, now + windowMs)
|
|
205
|
+
.exec()
|
|
206
|
+
|
|
207
|
+
// ioredis returns [[null, result], [null, result]] format
|
|
208
|
+
const count = (result?.[0]?.[1] as number) ?? 1
|
|
209
|
+
|
|
210
|
+
return {
|
|
211
|
+
count,
|
|
212
|
+
resetTime: now + windowMs,
|
|
213
|
+
}
|
|
214
|
+
} catch (err) {
|
|
215
|
+
logger.error('Redis increment error', { key }, err instanceof Error ? err : undefined)
|
|
216
|
+
return this.fallback.increment(key, windowMs)
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
async destroy(): Promise<void> {
|
|
221
|
+
this.fallback.destroy()
|
|
222
|
+
if (this.redis) {
|
|
223
|
+
try {
|
|
224
|
+
await this.redis.disconnect()
|
|
225
|
+
} catch {
|
|
226
|
+
// Ignore disconnect errors
|
|
227
|
+
}
|
|
228
|
+
this.redis = null
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
// Type for ioredis client (minimal interface)
|
|
234
|
+
interface RedisClient {
|
|
235
|
+
status: 'ready' | 'connecting' | 'reconnecting' | 'end'
|
|
236
|
+
on(event: string, callback: (arg: unknown) => void): void
|
|
237
|
+
disconnect(): Promise<void>
|
|
238
|
+
get(key: string): Promise<string | null>
|
|
239
|
+
set(key: string, value: string, ...args: (string | number)[]): Promise<'OK' | null>
|
|
240
|
+
incr(key: string): RedisChainable
|
|
241
|
+
pexpireat(key: string, timestamp: number): RedisChainable
|
|
242
|
+
multi(): RedisChainable
|
|
243
|
+
exec(): Promise<Array<[Error | null, unknown]> | null>
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
// Type for ioredis chainable commands
|
|
247
|
+
interface RedisChainable {
|
|
248
|
+
incr(key: string): RedisChainable
|
|
249
|
+
pexpireat(key: string, timestamp: number): RedisChainable
|
|
250
|
+
exec(): Promise<Array<[Error | null, unknown]> | null>
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
interface RateLimitConfig {
|
|
254
|
+
windowMs: number
|
|
255
|
+
maxRequests: number
|
|
256
|
+
keyGenerator?: (c: Context) => string
|
|
257
|
+
store?: RateLimitStore
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
const DEFAULT_CONFIG: RateLimitConfig = {
|
|
261
|
+
windowMs: 60 * 1000, // 1 minute
|
|
262
|
+
maxRequests: 100,
|
|
263
|
+
}
|
|
264
|
+
|
|
265
|
+
// Global store instance - uses Redis if available, falls back to memory
|
|
266
|
+
let globalStore: RateLimitStore | null = null
|
|
267
|
+
|
|
268
|
+
/**
|
|
269
|
+
* Get or create the global rate limit store.
|
|
270
|
+
* Uses Redis if REDIS_URL is set, otherwise uses in-memory store.
|
|
271
|
+
*/
|
|
272
|
+
function getGlobalStore(): RateLimitStore {
|
|
273
|
+
if (!globalStore) {
|
|
274
|
+
if (process.env.REDIS_URL) {
|
|
275
|
+
globalStore = new RedisRateLimitStore()
|
|
276
|
+
} else {
|
|
277
|
+
globalStore = new MemoryRateLimitStore()
|
|
278
|
+
}
|
|
279
|
+
}
|
|
280
|
+
return globalStore
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
/**
|
|
284
|
+
* Rate limiting middleware.
|
|
285
|
+
* Limits requests to maxRequests per windowMs per client.
|
|
286
|
+
*
|
|
287
|
+
* Supports Redis for distributed deployments (set REDIS_URL env var).
|
|
288
|
+
* Falls back to in-memory store for single-instance deployments.
|
|
289
|
+
*/
|
|
290
|
+
export const rateLimitMiddleware = (config: Partial<RateLimitConfig> = {}): MiddlewareHandler => {
|
|
291
|
+
const { windowMs, maxRequests, keyGenerator } = { ...DEFAULT_CONFIG, ...config }
|
|
292
|
+
const store = config.store ?? getGlobalStore()
|
|
293
|
+
|
|
294
|
+
return async (c: Context, next) => {
|
|
295
|
+
// Generate a unique key for the client
|
|
296
|
+
const key = keyGenerator ? keyGenerator(c) : getClientKey(c)
|
|
297
|
+
|
|
298
|
+
const now = Date.now()
|
|
299
|
+
|
|
300
|
+
// Increment counter atomically
|
|
301
|
+
const entry = await store.increment(key, windowMs)
|
|
302
|
+
|
|
303
|
+
// Calculate remaining requests and time
|
|
304
|
+
const remaining = Math.max(0, maxRequests - entry.count)
|
|
305
|
+
const resetSeconds = Math.ceil((entry.resetTime - now) / 1000)
|
|
306
|
+
|
|
307
|
+
// Set rate limit headers
|
|
308
|
+
c.header('X-RateLimit-Limit', String(maxRequests))
|
|
309
|
+
c.header('X-RateLimit-Remaining', String(remaining))
|
|
310
|
+
c.header('X-RateLimit-Reset', String(resetSeconds))
|
|
311
|
+
|
|
312
|
+
// Check if rate limit exceeded
|
|
313
|
+
if (entry.count > maxRequests) {
|
|
314
|
+
c.header('Retry-After', String(resetSeconds))
|
|
315
|
+
|
|
316
|
+
return c.json(
|
|
317
|
+
{
|
|
318
|
+
error: {
|
|
319
|
+
code: ErrorCodes.RATE_LIMITED,
|
|
320
|
+
message: `Rate limit exceeded. Try again in ${resetSeconds} seconds`,
|
|
321
|
+
},
|
|
322
|
+
status: 429,
|
|
323
|
+
},
|
|
324
|
+
429
|
|
325
|
+
)
|
|
326
|
+
}
|
|
327
|
+
|
|
328
|
+
return next()
|
|
329
|
+
}
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
/**
|
|
333
|
+
* Generates a unique key for rate limiting based on the client.
|
|
334
|
+
* Uses API key if available, otherwise falls back to IP address.
|
|
335
|
+
*/
|
|
336
|
+
function getClientKey(c: Context): string {
|
|
337
|
+
// Try to use the authenticated user's API key
|
|
338
|
+
const auth = c.get('auth')
|
|
339
|
+
if (auth?.apiKey) {
|
|
340
|
+
return `api:${auth.apiKey}`
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
// Fall back to IP address
|
|
344
|
+
const forwarded = c.req.header('x-forwarded-for')
|
|
345
|
+
const ip = forwarded?.split(',')[0]?.trim() || c.req.header('x-real-ip') || 'unknown'
|
|
346
|
+
return `ip:${ip}`
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Creates a rate limiter with custom settings per endpoint.
|
|
351
|
+
*/
|
|
352
|
+
export const createRateLimiter = (maxRequests: number, windowMs: number = 60000): MiddlewareHandler => {
|
|
353
|
+
return rateLimitMiddleware({ maxRequests, windowMs })
|
|
354
|
+
}
|
|
355
|
+
|
|
356
|
+
// Pre-configured rate limiters for different use cases
|
|
357
|
+
export const standardRateLimit = rateLimitMiddleware() // 100 req/min
|
|
358
|
+
export const strictRateLimit = rateLimitMiddleware({ maxRequests: 20 }) // 20 req/min
|
|
359
|
+
export const searchRateLimit = rateLimitMiddleware({ maxRequests: 50 }) // 50 req/min
|
|
360
|
+
export const uploadRateLimit = rateLimitMiddleware({ maxRequests: 10 }) // 10 req/min
|