@twelvehart/supermemory-runtime 1.0.0-next.0

package/src/mcp/results.ts

@@ -0,0 +1,104 @@
+import { ErrorCode as McpProtocolErrorCode, McpError, type CallToolResult } from '@modelcontextprotocol/sdk/types.js'
+import { ZodError } from 'zod'
+import { AppError, ValidationError, ErrorCode as AppErrorCode } from '../utils/errors.js'
+
+export interface McpEnvelopeError {
+  code: string
+  message: string
+  details?: unknown
+  retriable?: boolean
+}
+
+export interface McpResultEnvelope<T> extends Record<string, unknown> {
+  ok: boolean
+  data: T | null
+  warnings: string[]
+  errors: McpEnvelopeError[]
+  meta: {
+    tool: string
+    timestamp: string
+    partial?: boolean
+  }
+}
+
+function extractHumanMessage<T>(toolName: string, data: T | null, ok: boolean): string {
+  if (data && typeof data === 'object' && data !== null && 'message' in data && typeof data.message === 'string') {
+    return data.message
+  }
+
+  return ok ? `${toolName} completed successfully.` : `${toolName} completed with errors.`
+}
+
+function formatValidationMessage(error: ValidationError): string {
+  return `${error.message} | fieldErrors=${JSON.stringify(error.fieldErrors)}`
+}
+
+export function createMcpEnvelopeError(
+  code: string,
+  message: string,
+  details?: unknown,
+  retriable?: boolean
+): McpEnvelopeError {
+  return { code, message, details, retriable }
+}
+
+export function createToolResponse<T>(options: {
+  data: T | null
+  errors?: McpEnvelopeError[]
+  ok: boolean
+  tool: string
+  warnings?: string[]
+  partial?: boolean
+}): CallToolResult {
+  const warnings = options.warnings ?? []
+  const errors = options.errors ?? []
+  const envelope: McpResultEnvelope<T> = {
+    ok: options.ok,
+    data: options.data,
+    warnings,
+    errors,
+    meta: {
+      tool: options.tool,
+      timestamp: new Date().toISOString(),
+      ...(options.partial ? { partial: true } : {}),
+    },
+  }
+
+  const text = `${extractHumanMessage(options.tool, options.data, options.ok)}\n\n${JSON.stringify(envelope, null, 2)}`
+
+  return {
+    content: [{ type: 'text', text }],
+    structuredContent: envelope,
+    ...(options.ok && errors.length === 0 ? {} : { isError: true }),
+  }
+}
+
+export function mapErrorToMcpError(error: unknown): McpError {
+  if (error instanceof McpError) {
+    return error
+  }
+
+  if (error instanceof ZodError) {
+    const validationError = ValidationError.fromZodError(error)
+    return new McpError(McpProtocolErrorCode.InvalidParams, formatValidationMessage(validationError))
+  }
+
+  if (error instanceof ValidationError) {
+    return new McpError(McpProtocolErrorCode.InvalidParams, formatValidationMessage(error))
+  }
+
+  if (error instanceof AppError) {
+    if (error.code === AppErrorCode.NOT_FOUND || error.statusCode === 404) {
+      return new McpError(McpProtocolErrorCode.InvalidRequest, error.message)
+    }
+
+    if (error.statusCode >= 400 && error.statusCode < 500) {
+      return new McpError(McpProtocolErrorCode.InvalidParams, error.message)
+    }
+
+    return new McpError(McpProtocolErrorCode.InternalError, error.message)
+  }
+
+  const message = error instanceof Error ? error.message : 'Unknown error'
+  return new McpError(McpProtocolErrorCode.InternalError, message)
+}

package/src/mcp/tools.ts

@@ -0,0 +1,401 @@
+/**
+ * MCP Tool Definitions for Supermemory
+ */
+
+import { z } from 'zod'
+
+const MAX_CONTENT_CHARS = 50000
+const MAX_QUERY_CHARS = 10000
+const MAX_CONTAINER_TAG_CHARS = 100
+const MAX_TITLE_CHARS = 500
+const MAX_FACT_CHARS = 5000
+const MAX_METADATA_BYTES = 10240
+
+const boundedMetadataSchema = z
+  .record(z.unknown())
+  .optional()
+  .refine(
+    (metadata) => {
+      if (!metadata) return true
+      try {
+        const jsonSize = new TextEncoder().encode(JSON.stringify(metadata)).length
+        return jsonSize <= MAX_METADATA_BYTES
+      } catch {
+        return false
+      }
+    },
+    { message: `Metadata must be at most ${MAX_METADATA_BYTES} bytes (10KB)` }
+  )
+
+export const AddContentInputSchema = z
+  .object({
+    content: z
+      .string()
+      .min(1, 'Content is required')
+      .max(MAX_CONTENT_CHARS, `Content must be at most ${MAX_CONTENT_CHARS} characters`),
+    customId: z
+      .string()
+      .min(1, 'customId cannot be empty')
+      .max(255, 'customId must be at most 255 characters')
+      .optional(),
+    idempotencyKey: z
+      .string()
+      .min(1, 'idempotencyKey cannot be empty')
+      .max(255, 'idempotencyKey must be at most 255 characters')
+      .optional(),
+    containerTag: z
+      .string()
+      .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+      .regex(/^[a-zA-Z0-9_-]+$/, 'Container tag can only contain alphanumeric characters, underscores, and hyphens')
+      .optional()
+      .describe('Container/namespace for organizing memories'),
+    metadata: boundedMetadataSchema.describe('Additional metadata to attach (max 10KB)'),
+    sourceUrl: z
+      .string()
+      .url()
+      .refine(
+        (url) => {
+          try {
+            const parsed = new URL(url)
+            return ['http:', 'https:'].includes(parsed.protocol)
+          } catch {
+            return false
+          }
+        },
+        { message: 'URL must use http or https protocol' }
+      )
+      .optional(),
+    title: z.string().max(MAX_TITLE_CHARS, `Title must be at most ${MAX_TITLE_CHARS} characters`).optional(),
+    upsert: z.boolean().optional().default(false),
+  })
+  .superRefine((input, ctx) => {
+    if (input.customId && input.idempotencyKey && input.customId !== input.idempotencyKey) {
+      ctx.addIssue({
+        code: z.ZodIssueCode.custom,
+        message: 'customId and idempotencyKey must match when both are provided',
+        path: ['idempotencyKey'],
+      })
+    }
+  })
+
+export const SearchInputSchema = z.object({
+  query: z
+    .string()
+    .min(1, 'Query is required')
+    .max(MAX_QUERY_CHARS, `Query must be at most ${MAX_QUERY_CHARS} characters`),
+  containerTag: z
+    .string()
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+    .optional(),
+  mode: z.enum(['vector', 'memory', 'hybrid']).optional().default('hybrid'),
+  limit: z.number().min(1).max(100).optional().default(10),
+  threshold: z.number().min(0).max(1).optional().default(0.5),
+  rerank: z.boolean().optional().default(false),
+  includeMetadata: z.boolean().optional().default(true),
+})
+
+export const ProfileInputSchema = z.object({
+  containerTag: z
+    .string()
+    .min(1, 'Container tag is required')
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`),
+  action: z.enum(['get', 'update', 'ingest']).optional().default('get'),
+  content: z.string().max(MAX_CONTENT_CHARS, `Content must be at most ${MAX_CONTENT_CHARS} characters`).optional(),
+  facts: z
+    .array(
+      z.object({
+        content: z
+          .string()
+          .min(1, 'Fact content is required')
+          .max(MAX_FACT_CHARS, `Fact must be at most ${MAX_FACT_CHARS} characters`),
+        type: z.enum(['static', 'dynamic']).optional(),
+        category: z.string().max(100, 'Category must be at most 100 characters').optional(),
+      })
+    )
+    .max(100, 'Cannot add more than 100 facts at once')
+    .optional(),
+})
+
+export const ListDocumentsInputSchema = z.object({
+  containerTag: z
+    .string()
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+    .optional(),
+  limit: z.number().min(1).max(100).optional().default(20),
+  offset: z.number().min(0).optional().default(0),
+  contentType: z.enum(['note', 'url', 'pdf', 'image', 'tweet', 'document']).optional(),
+  sortBy: z.enum(['createdAt', 'updatedAt', 'title']).optional().default('createdAt'),
+  sortOrder: z.enum(['asc', 'desc']).optional().default('desc'),
+})
+
+export const DeleteContentInputSchema = z.object({
+  id: z.string().max(255, 'ID must be at most 255 characters').optional(),
+  containerTag: z
+    .string()
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+    .optional(),
+  confirm: z.boolean(),
+})
+
+export const RememberInputSchema = z.object({
+  fact: z.string().min(1, 'Fact is required').max(MAX_FACT_CHARS, `Fact must be at most ${MAX_FACT_CHARS} characters`),
+  containerTag: z
+    .string()
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+    .optional()
+    .default('default'),
+  type: z.enum(['static', 'dynamic']).optional().default('static'),
+  category: z
+    .enum(['identity', 'preference', 'skill', 'background', 'relationship', 'project', 'goal', 'context', 'other'])
+    .optional(),
+  expirationHours: z.number().min(1).max(8760).optional(),
+})
+
+export const RecallInputSchema = z.object({
+  query: z
+    .string()
+    .min(1, 'Query is required')
+    .max(MAX_QUERY_CHARS, `Query must be at most ${MAX_QUERY_CHARS} characters`),
+  containerTag: z
+    .string()
+    .max(MAX_CONTAINER_TAG_CHARS, `Container tag must be at most ${MAX_CONTAINER_TAG_CHARS} characters`)
+    .optional()
+    .default('default'),
+  includeStatic: z.boolean().optional().default(true),
+  includeDynamic: z.boolean().optional().default(true),
+  limit: z.number().min(1).max(50).optional().default(10),
+})
+
+export interface ToolDefinition {
+  name: string
+  description: string
+  inputSchema: {
+    type: 'object'
+    properties: Record<string, unknown>
+    required?: string[]
+  }
+}
+
+export const TOOL_DEFINITIONS: ToolDefinition[] = [
+  {
+    name: 'supermemory_add',
+    description: 'Add content and extract/index memories.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        content: { type: 'string' },
+        customId: { type: 'string' },
+        idempotencyKey: { type: 'string' },
+        containerTag: { type: 'string' },
+        metadata: { type: 'object', additionalProperties: true },
+        sourceUrl: { type: 'string' },
+        title: { type: 'string' },
+        upsert: { type: 'boolean' },
+      },
+      required: ['content'],
+    },
+  },
+  {
+    name: 'supermemory_search',
+    description: 'Semantic/memory/hybrid search across stored memories.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string' },
+        containerTag: { type: 'string' },
+        mode: { type: 'string', enum: ['vector', 'memory', 'hybrid'] },
+        limit: { type: 'number' },
+        threshold: { type: 'number' },
+        rerank: { type: 'boolean' },
+      },
+      required: ['query'],
+    },
+  },
+  {
+    name: 'supermemory_profile',
+    description: 'Get/ingest/update profile facts for a container.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        containerTag: { type: 'string' },
+        action: { type: 'string', enum: ['get', 'update', 'ingest'] },
+        content: { type: 'string' },
+        facts: {
+          type: 'array',
+          items: {
+            type: 'object',
+            properties: {
+              content: { type: 'string' },
+              type: { type: 'string', enum: ['static', 'dynamic'] },
+              category: { type: 'string' },
+            },
+            required: ['content'],
+          },
+        },
+      },
+      required: ['containerTag'],
+    },
+  },
+  {
+    name: 'supermemory_list',
+    description: 'List stored documents.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        containerTag: { type: 'string' },
+        limit: { type: 'number' },
+        offset: { type: 'number' },
+        contentType: { type: 'string', enum: ['note', 'url', 'pdf', 'image', 'tweet', 'document'] },
+        sortBy: { type: 'string', enum: ['createdAt', 'updatedAt', 'title'] },
+        sortOrder: { type: 'string', enum: ['asc', 'desc'] },
+      },
+    },
+  },
+  {
+    name: 'supermemory_delete',
+    description: 'Delete content by ID or containerTag.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        id: { type: 'string' },
+        containerTag: { type: 'string' },
+        confirm: { type: 'boolean' },
+      },
+      required: ['confirm'],
+    },
+  },
+  {
+    name: 'supermemory_remember',
+    description: 'Store a specific fact for future recall.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        fact: { type: 'string' },
+        containerTag: { type: 'string' },
+        type: { type: 'string', enum: ['static', 'dynamic'] },
+        category: {
+          type: 'string',
+          enum: [
+            'identity',
+            'preference',
+            'skill',
+            'background',
+            'relationship',
+            'project',
+            'goal',
+            'context',
+            'other',
+          ],
+        },
+        expirationHours: { type: 'number' },
+      },
+      required: ['fact'],
+    },
+  },
+  {
+    name: 'supermemory_recall',
+    description: 'Recall stored facts matching a query.',
+    inputSchema: {
+      type: 'object',
+      properties: {
+        query: { type: 'string' },
+        containerTag: { type: 'string' },
+        includeStatic: { type: 'boolean' },
+        includeDynamic: { type: 'boolean' },
+        limit: { type: 'number' },
+      },
+      required: ['query'],
+    },
+  },
+]
+
+export interface AddContentResult {
+  success: boolean
+  documentId?: string
+  customId?: string
+  created?: boolean
+  reused?: boolean
+  updated?: boolean
+  memoriesExtracted?: number
+  message: string
+  errors?: string[]
+}
+
+export interface SearchResultItem {
+  id: string
+  content: string
+  similarity: number
+  containerTag?: string
+  metadata?: Record<string, unknown>
+  createdAt?: string
+}
+
+export interface SearchResult {
+  results: SearchResultItem[]
+  totalCount: number
+  query: string
+  searchTimeMs: number
+}
+
+export interface ProfileResult {
+  containerTag: string
+  staticFacts: Array<{
+    id: string
+    content: string
+    category?: string
+    confidence: number
+  }>
+  dynamicFacts: Array<{
+    id: string
+    content: string
+    category?: string
+    expiresAt?: string
+  }>
+  lastUpdated: string
+}
+
+export interface ListResult {
+  documents: Array<{
+    id: string
+    title?: string
+    contentPreview: string
+    contentType: string
+    containerTag?: string
+    createdAt: string
+    updatedAt: string
+  }>
+  total: number
+  limit: number
+  offset: number
+  hasMore: boolean
+}
+
+export interface DeleteResult {
+  success: boolean
+  documentsDeleted: number
+  memoriesDeleted: number
+  vectorsDeleted: number
+  profileFactsDeleted: number
+  deletedCount: number
+  message: string
+  errors?: string[]
+}
+
+export interface RememberResult {
+  success: boolean
+  factId: string
+  message: string
+}
+
+export interface RecallResult {
+  facts: Array<{
+    id: string
+    content: string
+    type: 'static' | 'dynamic'
+    category?: string
+    confidence: number
+    createdAt: string
+  }>
+  query: string
+  totalFound: number
+}

package/src/queues/config.ts

@@ -0,0 +1,119 @@
+/**
+ * BullMQ Queue Configuration
+ *
+ * Configures Redis connection, queue options, retry logic, and concurrency settings
+ * for the job queue system.
+ */
+
+import { QueueOptions, WorkerOptions } from 'bullmq'
+
+/**
+ * Redis connection configuration with health checks and reconnection logic
+ */
+export const redisConfig = {
+  connection: {
+    host: process.env.REDIS_HOST || 'localhost',
+    port: parseInt(process.env.REDIS_PORT || '6379', 10),
+    maxRetriesPerRequest: null, // Required for BullMQ
+    enableReadyCheck: true,
+    enableOfflineQueue: true,
+    retryStrategy: (times: number) => {
+      // Exponential backoff: 1s, 2s, 4s, 8s, max 30s
+      const delay = Math.min(times * 1000, 30000)
+      console.log(`[Redis] Reconnection attempt ${times}, waiting ${delay}ms`)
+      return delay
+    },
+    reconnectOnError: (err: Error) => {
+      const targetErrors = ['READONLY', 'ECONNREFUSED', 'ETIMEDOUT']
+      if (targetErrors.some((targetError) => err.message.includes(targetError))) {
+        console.log(`[Redis] Reconnecting due to error: ${err.message}`)
+        return true
+      }
+      return false
+    },
+  },
+}
+
+/**
+ * Default queue options with retry logic and dead letter queue support
+ */
+export const defaultQueueOptions: QueueOptions = {
+  connection: redisConfig.connection,
+  defaultJobOptions: {
+    attempts: 3, // Maximum retry attempts
+    backoff: {
+      type: 'exponential',
+      delay: 1000, // Initial delay: 1 second
+    },
+    removeOnComplete: {
+      count: 100, // Keep last 100 completed jobs
+      age: 24 * 3600, // Keep for 24 hours
+    },
+    removeOnFail: {
+      count: 500, // Keep last 500 failed jobs for debugging
+      age: 7 * 24 * 3600, // Keep for 7 days
+    },
+  },
+}
+
+/**
+ * Queue-specific concurrency settings from environment variables
+ */
+export const concurrencySettings = {
+  extraction: parseInt(process.env.BULLMQ_CONCURRENCY_EXTRACTION || '5', 10),
+  chunking: parseInt(process.env.BULLMQ_CONCURRENCY_CHUNKING || '3', 10),
+  embedding: parseInt(process.env.BULLMQ_CONCURRENCY_EMBEDDING || '2', 10),
+  indexing: parseInt(process.env.BULLMQ_CONCURRENCY_INDEXING || '1', 10),
+}
+
+/**
+ * Worker options factory
+ * Creates worker configuration with concurrency settings
+ */
+export function createWorkerOptions(queueName: keyof typeof concurrencySettings): WorkerOptions {
+  return {
+    connection: redisConfig.connection,
+    concurrency: concurrencySettings[queueName],
+    autorun: true,
+    removeOnComplete: { count: 100 },
+    removeOnFail: { count: 500 },
+  }
+}
+
+/**
+ * Dead letter queue configuration
+ * Failed jobs after max retries are moved here for manual inspection
+ */
+export const deadLetterQueueOptions: QueueOptions = {
+  ...defaultQueueOptions,
+  defaultJobOptions: {
+    attempts: 1, // No retries for dead letter queue
+    removeOnComplete: {
+      count: 1000,
+      age: 30 * 24 * 3600, // Keep for 30 days
+    },
+    removeOnFail: false, // Never remove failed jobs from DLQ
+  },
+}
+
+/**
+ * Priority levels for job scheduling
+ * Higher number = higher priority (executed first)
+ */
+export enum JobPriority {
+  LOW = 1,
+  NORMAL = 5,
+  HIGH = 8,
+  CRITICAL = 10,
+}
+
+/**
+ * Job progress tracking interface
+ */
+export interface JobProgress {
+  percentage: number // 0-100
+  stage?: string
+  message?: string
+  processedItems?: number
+  totalItems?: number
+}