@stackmemoryai/stackmemory 0.3.1 → 0.3.5

package/dist/core/storage/chromadb-adapter.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/core/storage/chromadb-adapter.ts"],
+  "sourcesContent": ["/**\n * ChromaDB Storage Adapter for StackMemory\n * \n * Provides vector storage and semantic search capabilities for context data\n * using ChromaDB cloud service with user and team segmentation.\n */\n\nimport { CloudClient, Collection } from 'chromadb';\nimport { v4 as uuidv4 } from 'uuid';\nimport { Frame } from '../context/frame-manager.js';\nimport { Logger } from '../monitoring/logger.js';\n\ninterface ChromaDocument {\n  id: string;\n  document: string;\n  metadata: {\n    user_id: string;\n    team_id?: string;\n    frame_id: string;\n    session_id: string;\n    project_name: string;\n    timestamp: number;\n    type: 'frame' | 'decision' | 'observation' | 'context';\n    score?: number;\n    tags?: string[];\n  };\n}\n\ninterface ChromaConfig {\n  apiKey: string;\n  tenant: string;\n  database: string;\n  collectionName?: string;\n}\n\nexport class ChromaDBAdapter {\n  private client: CloudClient;\n  private collection: Collection | null = null;\n  private logger: Logger;\n  private config: ChromaConfig;\n  private userId: string;\n  private teamId?: string;\n\n  constructor(config: ChromaConfig, userId: string, teamId?: string) {\n    this.config = config;\n    this.userId = userId;\n    this.teamId = teamId;\n    this.logger = new Logger('ChromaDBAdapter');\n\n    // Initialize ChromaDB client\n    this.client = new CloudClient({\n      apiKey: config.apiKey,\n      tenant: config.tenant,\n      database: config.database,\n    });\n  }\n\n  async initialize(): Promise<void> {\n    try {\n      const collectionName = this.config.collectionName || 'stackmemory_contexts';\n      \n      // Get or create collection with metadata for filtering\n      this.collection = await this.client.getOrCreateCollection({\n        name: collectionName,\n        metadata: {\n          description: 'StackMemory context storage',\n          version: '1.0.0',\n          created_at: new Date().toISOString(),\n        },\n      });\n\n      this.logger.info(`ChromaDB collection '${collectionName}' initialized`);\n    } catch (error) {\n      this.logger.error('Failed to initialize ChromaDB collection', error);\n      throw error;\n    }\n  }\n\n  /**\n   * Store a frame in ChromaDB\n   */\n  async storeFrame(frame: Frame): Promise<void> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      // Prepare document from frame\n      const frameMetadata: any = {\n        user_id: this.userId,\n        frame_id: frame.frameId,\n        session_id: frame.sessionId || 'unknown',\n        project_name: frame.projectName || 'default',\n        timestamp: frame.timestamp,\n        type: 'frame',\n        score: frame.score,\n        tags: frame.tags || [],\n      };\n\n      // Only add team_id if it exists\n      if (this.teamId) {\n        frameMetadata.team_id = this.teamId;\n      }\n\n      const document: ChromaDocument = {\n        id: `frame_${frame.frameId}_${this.userId}`,\n        document: this.frameToDocument(frame),\n        metadata: frameMetadata,\n      };\n\n      // Add to ChromaDB\n      await this.collection.add({\n        ids: [document.id],\n        documents: [document.document],\n        metadatas: [document.metadata],\n      });\n\n      this.logger.debug(`Stored frame ${frame.frameId} for user ${this.userId}`);\n    } catch (error) {\n      this.logger.error(`Failed to store frame ${frame.frameId}`, error);\n      throw error;\n    }\n  }\n\n  /**\n   * Store a decision or observation\n   */\n  async storeContext(\n    type: 'decision' | 'observation',\n    content: string,\n    metadata?: Record<string, any>\n  ): Promise<void> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      const contextId = `${type}_${uuidv4()}_${this.userId}`;\n      \n      const documentMetadata: any = {\n        user_id: this.userId,\n        frame_id: metadata?.frame_id || 'none',\n        session_id: metadata?.session_id || 'unknown',\n        project_name: metadata?.project_name || 'default',\n        timestamp: Date.now(),\n        type,\n        ...metadata,\n      };\n\n      // Only add team_id if it exists (ChromaDB doesn't accept undefined values)\n      if (this.teamId) {\n        documentMetadata.team_id = this.teamId;\n      }\n\n      const document: ChromaDocument = {\n        id: contextId,\n        document: content,\n        metadata: documentMetadata,\n      };\n\n      await this.collection.add({\n        ids: [document.id],\n        documents: [document.document],\n        metadatas: [document.metadata],\n      });\n\n      this.logger.debug(`Stored ${type} for user ${this.userId}`);\n    } catch (error) {\n      this.logger.error(`Failed to store ${type}`, error);\n      throw error;\n    }\n  }\n\n  /**\n   * Query contexts by semantic similarity\n   */\n  async queryContexts(\n    query: string,\n    limit: number = 10,\n    filters?: {\n      type?: string[];\n      projectName?: string;\n      sessionId?: string;\n      startTime?: number;\n      endTime?: number;\n    }\n  ): Promise<Array<{ content: string; metadata: any; distance: number }>> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      // Build where clause for filtering\n      const whereClause: any = {\n        user_id: this.userId,\n      };\n\n      // Add team filter if applicable\n      if (this.teamId) {\n        whereClause['$or'] = [\n          { team_id: this.teamId },\n          { user_id: this.userId },\n        ];\n      }\n\n      // Add additional filters\n      if (filters?.type && filters.type.length > 0) {\n        whereClause.type = { $in: filters.type };\n      }\n\n      if (filters?.projectName) {\n        whereClause.project_name = filters.projectName;\n      }\n\n      if (filters?.sessionId) {\n        whereClause.session_id = filters.sessionId;\n      }\n\n      if (filters?.startTime || filters?.endTime) {\n        whereClause.timestamp = {};\n        if (filters.startTime) {\n          whereClause.timestamp.$gte = filters.startTime;\n        }\n        if (filters.endTime) {\n          whereClause.timestamp.$lte = filters.endTime;\n        }\n      }\n\n      // Query ChromaDB\n      const results = await this.collection.query({\n        queryTexts: [query],\n        nResults: limit,\n        where: whereClause,\n        include: ['documents', 'metadatas', 'distances'],\n      });\n\n      // Format results\n      const contexts: Array<{ content: string; metadata: any; distance: number }> = [];\n      \n      if (results.documents && results.documents[0]) {\n        for (let i = 0; i < results.documents[0].length; i++) {\n          contexts.push({\n            content: results.documents[0][i] || '',\n            metadata: results.metadatas?.[0]?.[i] || {},\n            distance: results.distances?.[0]?.[i] || 0,\n          });\n        }\n      }\n\n      this.logger.debug(`Found ${contexts.length} contexts for query`);\n      return contexts;\n    } catch (error) {\n      this.logger.error('Failed to query contexts', error);\n      throw error;\n    }\n  }\n\n  /**\n   * Get user's recent contexts\n   */\n  async getRecentContexts(\n    limit: number = 20,\n    type?: string\n  ): Promise<Array<{ content: string; metadata: any }>> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      const whereClause: any = {\n        user_id: this.userId,\n      };\n\n      if (type) {\n        whereClause.type = type;\n      }\n\n      // Get all documents for the user (ChromaDB doesn't support direct ordering)\n      const results = await this.collection.get({\n        where: whereClause,\n        include: ['documents', 'metadatas'],\n      });\n\n      // Sort by timestamp and limit\n      const contexts: Array<{ content: string; metadata: any }> = [];\n      \n      if (results.documents) {\n        const indexed = results.documents.map((doc, i) => ({\n          content: doc || '',\n          metadata: results.metadatas?.[i] || {},\n        }));\n\n        // Sort by timestamp descending\n        indexed.sort((a, b) => (b.metadata.timestamp || 0) - (a.metadata.timestamp || 0));\n\n        // Take limit\n        contexts.push(...indexed.slice(0, limit));\n      }\n\n      return contexts;\n    } catch (error) {\n      this.logger.error('Failed to get recent contexts', error);\n      throw error;\n    }\n  }\n\n  /**\n   * Delete old contexts (retention policy)\n   */\n  async deleteOldContexts(olderThanDays: number = 30): Promise<number> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      const cutoffTime = Date.now() - (olderThanDays * 24 * 60 * 60 * 1000);\n\n      // Get old documents\n      const results = await this.collection.get({\n        where: {\n          user_id: this.userId,\n          timestamp: { $lt: cutoffTime },\n        },\n        include: ['ids'],\n      });\n\n      if (!results.ids || results.ids.length === 0) {\n        return 0;\n      }\n\n      // Delete old documents\n      await this.collection.delete({\n        ids: results.ids,\n      });\n\n      this.logger.info(`Deleted ${results.ids.length} old contexts`);\n      return results.ids.length;\n    } catch (error) {\n      this.logger.error('Failed to delete old contexts', error);\n      throw error;\n    }\n  }\n\n  /**\n   * Get team contexts (if user is part of a team)\n   */\n  async getTeamContexts(\n    limit: number = 20\n  ): Promise<Array<{ content: string; metadata: any }>> {\n    if (!this.collection || !this.teamId) {\n      return [];\n    }\n\n    try {\n      const results = await this.collection.get({\n        where: {\n          team_id: this.teamId,\n        },\n        include: ['documents', 'metadatas'],\n        limit,\n      });\n\n      const contexts: Array<{ content: string; metadata: any }> = [];\n      \n      if (results.documents) {\n        for (let i = 0; i < results.documents.length; i++) {\n          contexts.push({\n            content: results.documents[i] || '',\n            metadata: results.metadatas?.[i] || {},\n          });\n        }\n      }\n\n      return contexts;\n    } catch (error) {\n      this.logger.error('Failed to get team contexts', error);\n      return [];\n    }\n  }\n\n  /**\n   * Convert frame to searchable document\n   */\n  private frameToDocument(frame: Frame): string {\n    const parts = [\n      `Frame: ${frame.title}`,\n      `Type: ${frame.type}`,\n      `Status: ${frame.status}`,\n    ];\n\n    if (frame.description) {\n      parts.push(`Description: ${frame.description}`);\n    }\n\n    if (frame.inputs && frame.inputs.length > 0) {\n      parts.push(`Inputs: ${frame.inputs.join(', ')}`);\n    }\n\n    if (frame.outputs && frame.outputs.length > 0) {\n      parts.push(`Outputs: ${frame.outputs.join(', ')}`);\n    }\n\n    if (frame.tags && frame.tags.length > 0) {\n      parts.push(`Tags: ${frame.tags.join(', ')}`);\n    }\n\n    if (frame.digest_json) {\n      try {\n        const digest = JSON.parse(frame.digest_json);\n        if (digest.summary) {\n          parts.push(`Summary: ${digest.summary}`);\n        }\n        if (digest.keyDecisions) {\n          parts.push(`Decisions: ${digest.keyDecisions.join('. ')}`);\n        }\n      } catch {\n        // Ignore parse errors\n      }\n    }\n\n    return parts.join('\\n');\n  }\n\n  /**\n   * Update team ID for a user\n   */\n  async updateTeamId(newTeamId: string): Promise<void> {\n    this.teamId = newTeamId;\n    this.logger.info(`Updated team ID to ${newTeamId} for user ${this.userId}`);\n  }\n\n  /**\n   * Get storage statistics\n   */\n  async getStats(): Promise<{\n    totalDocuments: number;\n    userDocuments: number;\n    teamDocuments?: number;\n    documentsByType: Record<string, number>;\n  }> {\n    if (!this.collection) {\n      throw new Error('ChromaDB not initialized');\n    }\n\n    try {\n      // Get user documents\n      const userResults = await this.collection.get({\n        where: { user_id: this.userId },\n        include: ['metadatas'],\n      });\n\n      const stats: any = {\n        totalDocuments: 0,\n        userDocuments: userResults.ids?.length || 0,\n        documentsByType: {},\n      };\n\n      // Count by type\n      if (userResults.metadatas) {\n        for (const metadata of userResults.metadatas) {\n          const type = metadata?.type || 'unknown';\n          stats.documentsByType[type] = (stats.documentsByType[type] || 0) + 1;\n        }\n      }\n\n      // Get team documents if applicable\n      if (this.teamId) {\n        const teamResults = await this.collection.get({\n          where: { team_id: this.teamId },\n          include: ['ids'],\n        });\n        stats.teamDocuments = teamResults.ids?.length || 0;\n      }\n\n      stats.totalDocuments = stats.userDocuments + (stats.teamDocuments || 0);\n\n      return stats;\n    } catch (error) {\n      this.logger.error('Failed to get stats', error);\n      throw error;\n    }\n  }\n}"],
+  "mappings": "AAOA,SAAS,mBAA+B;AACxC,SAAS,MAAM,cAAc;AAE7B,SAAS,cAAc;AAyBhB,MAAM,gBAAgB;AAAA,EACnB;AAAA,EACA,aAAgC;AAAA,EAChC;AAAA,EACA;AAAA,EACA;AAAA,EACA;AAAA,EAER,YAAY,QAAsB,QAAgB,QAAiB;AACjE,SAAK,SAAS;AACd,SAAK,SAAS;AACd,SAAK,SAAS;AACd,SAAK,SAAS,IAAI,OAAO,iBAAiB;AAG1C,SAAK,SAAS,IAAI,YAAY;AAAA,MAC5B,QAAQ,OAAO;AAAA,MACf,QAAQ,OAAO;AAAA,MACf,UAAU,OAAO;AAAA,IACnB,CAAC;AAAA,EACH;AAAA,EAEA,MAAM,aAA4B;AAChC,QAAI;AACF,YAAM,iBAAiB,KAAK,OAAO,kBAAkB;AAGrD,WAAK,aAAa,MAAM,KAAK,OAAO,sBAAsB;AAAA,QACxD,MAAM;AAAA,QACN,UAAU;AAAA,UACR,aAAa;AAAA,UACb,SAAS;AAAA,UACT,aAAY,oBAAI,KAAK,GAAE,YAAY;AAAA,QACrC;AAAA,MACF,CAAC;AAED,WAAK,OAAO,KAAK,wBAAwB,cAAc,eAAe;AAAA,IACxE,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,4CAA4C,KAAK;AACnE,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,WAAW,OAA6B;AAC5C,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AAEF,YAAM,gBAAqB;AAAA,QACzB,SAAS,KAAK;AAAA,QACd,UAAU,MAAM;AAAA,QAChB,YAAY,MAAM,aAAa;AAAA,QAC/B,cAAc,MAAM,eAAe;AAAA,QACnC,WAAW,MAAM;AAAA,QACjB,MAAM;AAAA,QACN,OAAO,MAAM;AAAA,QACb,MAAM,MAAM,QAAQ,CAAC;AAAA,MACvB;AAGA,UAAI,KAAK,QAAQ;AACf,sBAAc,UAAU,KAAK;AAAA,MAC/B;AAEA,YAAM,WAA2B;AAAA,QAC/B,IAAI,SAAS,MAAM,OAAO,IAAI,KAAK,MAAM;AAAA,QACzC,UAAU,KAAK,gBAAgB,KAAK;AAAA,QACpC,UAAU;AAAA,MACZ;AAGA,YAAM,KAAK,WAAW,IAAI;AAAA,QACxB,KAAK,CAAC,SAAS,EAAE;AAAA,QACjB,WAAW,CAAC,SAAS,QAAQ;AAAA,QAC7B,WAAW,CAAC,SAAS,QAAQ;AAAA,MAC/B,CAAC;AAED,WAAK,OAAO,MAAM,gBAAgB,MAAM,OAAO,aAAa,KAAK,MAAM,EAAE;AAAA,IAC3E,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,yBAAyB,MAAM,OAAO,IAAI,KAAK;AACjE,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aACJ,MACA,SACA,UACe;AACf,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AACF,YAAM,YAAY,GAAG,IAAI,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM;AAEpD,YAAM,mBAAwB;AAAA,QAC5B,SAAS,KAAK;AAAA,QACd,UAAU,UAAU,YAAY;AAAA,QAChC,YAAY,UAAU,cAAc;AAAA,QACpC,cAAc,UAAU,gBAAgB;AAAA,QACxC,WAAW,KAAK,IAAI;AAAA,QACpB;AAAA,QACA,GAAG;AAAA,MACL;AAGA,UAAI,KAAK,QAAQ;AACf,yBAAiB,UAAU,KAAK;AAAA,MAClC;AAEA,YAAM,WAA2B;AAAA,QAC/B,IAAI;AAAA,QACJ,UAAU;AAAA,QACV,UAAU;AAAA,MACZ;AAEA,YAAM,KAAK,WAAW,IAAI;AAAA,QACxB,KAAK,CAAC,SAAS,EAAE;AAAA,QACjB,WAAW,CAAC,SAAS,QAAQ;AAAA,QAC7B,WAAW,CAAC,SAAS,QAAQ;AAAA,MAC/B,CAAC;AAED,WAAK,OAAO,MAAM,UAAU,IAAI,aAAa,KAAK,MAAM,EAAE;AAAA,IAC5D,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,mBAAmB,IAAI,IAAI,KAAK;AAClD,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,cACJ,OACA,QAAgB,IAChB,SAOsE;AACtE,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AAEF,YAAM,cAAmB;AAAA,QACvB,SAAS,KAAK;AAAA,MAChB;AAGA,UAAI,KAAK,QAAQ;AACf,oBAAY,KAAK,IAAI;AAAA,UACnB,EAAE,SAAS,KAAK,OAAO;AAAA,UACvB,EAAE,SAAS,KAAK,OAAO;AAAA,QACzB;AAAA,MACF;AAGA,UAAI,SAAS,QAAQ,QAAQ,KAAK,SAAS,GAAG;AAC5C,oBAAY,OAAO,EAAE,KAAK,QAAQ,KAAK;AAAA,MACzC;AAEA,UAAI,SAAS,aAAa;AACxB,oBAAY,eAAe,QAAQ;AAAA,MACrC;AAEA,UAAI,SAAS,WAAW;AACtB,oBAAY,aAAa,QAAQ;AAAA,MACnC;AAEA,UAAI,SAAS,aAAa,SAAS,SAAS;AAC1C,oBAAY,YAAY,CAAC;AACzB,YAAI,QAAQ,WAAW;AACrB,sBAAY,UAAU,OAAO,QAAQ;AAAA,QACvC;AACA,YAAI,QAAQ,SAAS;AACnB,sBAAY,UAAU,OAAO,QAAQ;AAAA,QACvC;AAAA,MACF;AAGA,YAAM,UAAU,MAAM,KAAK,WAAW,MAAM;AAAA,QAC1C,YAAY,CAAC,KAAK;AAAA,QAClB,UAAU;AAAA,QACV,OAAO;AAAA,QACP,SAAS,CAAC,aAAa,aAAa,WAAW;AAAA,MACjD,CAAC;AAGD,YAAM,WAAwE,CAAC;AAE/E,UAAI,QAAQ,aAAa,QAAQ,UAAU,CAAC,GAAG;AAC7C,iBAAS,IAAI,GAAG,IAAI,QAAQ,UAAU,CAAC,EAAE,QAAQ,KAAK;AACpD,mBAAS,KAAK;AAAA,YACZ,SAAS,QAAQ,UAAU,CAAC,EAAE,CAAC,KAAK;AAAA,YACpC,UAAU,QAAQ,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC;AAAA,YAC1C,UAAU,QAAQ,YAAY,CAAC,IAAI,CAAC,KAAK;AAAA,UAC3C,CAAC;AAAA,QACH;AAAA,MACF;AAEA,WAAK,OAAO,MAAM,SAAS,SAAS,MAAM,qBAAqB;AAC/D,aAAO;AAAA,IACT,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,4BAA4B,KAAK;AACnD,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,kBACJ,QAAgB,IAChB,MACoD;AACpD,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AACF,YAAM,cAAmB;AAAA,QACvB,SAAS,KAAK;AAAA,MAChB;AAEA,UAAI,MAAM;AACR,oBAAY,OAAO;AAAA,MACrB;AAGA,YAAM,UAAU,MAAM,KAAK,WAAW,IAAI;AAAA,QACxC,OAAO;AAAA,QACP,SAAS,CAAC,aAAa,WAAW;AAAA,MACpC,CAAC;AAGD,YAAM,WAAsD,CAAC;AAE7D,UAAI,QAAQ,WAAW;AACrB,cAAM,UAAU,QAAQ,UAAU,IAAI,CAAC,KAAK,OAAO;AAAA,UACjD,SAAS,OAAO;AAAA,UAChB,UAAU,QAAQ,YAAY,CAAC,KAAK,CAAC;AAAA,QACvC,EAAE;AAGF,gBAAQ,KAAK,CAAC,GAAG,OAAO,EAAE,SAAS,aAAa,MAAM,EAAE,SAAS,aAAa,EAAE;AAGhF,iBAAS,KAAK,GAAG,QAAQ,MAAM,GAAG,KAAK,CAAC;AAAA,MAC1C;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,iCAAiC,KAAK;AACxD,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,kBAAkB,gBAAwB,IAAqB;AACnE,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AACF,YAAM,aAAa,KAAK,IAAI,IAAK,gBAAgB,KAAK,KAAK,KAAK;AAGhE,YAAM,UAAU,MAAM,KAAK,WAAW,IAAI;AAAA,QACxC,OAAO;AAAA,UACL,SAAS,KAAK;AAAA,UACd,WAAW,EAAE,KAAK,WAAW;AAAA,QAC/B;AAAA,QACA,SAAS,CAAC,KAAK;AAAA,MACjB,CAAC;AAED,UAAI,CAAC,QAAQ,OAAO,QAAQ,IAAI,WAAW,GAAG;AAC5C,eAAO;AAAA,MACT;AAGA,YAAM,KAAK,WAAW,OAAO;AAAA,QAC3B,KAAK,QAAQ;AAAA,MACf,CAAC;AAED,WAAK,OAAO,KAAK,WAAW,QAAQ,IAAI,MAAM,eAAe;AAC7D,aAAO,QAAQ,IAAI;AAAA,IACrB,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,iCAAiC,KAAK;AACxD,YAAM;AAAA,IACR;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,gBACJ,QAAgB,IACoC;AACpD,QAAI,CAAC,KAAK,cAAc,CAAC,KAAK,QAAQ;AACpC,aAAO,CAAC;AAAA,IACV;AAEA,QAAI;AACF,YAAM,UAAU,MAAM,KAAK,WAAW,IAAI;AAAA,QACxC,OAAO;AAAA,UACL,SAAS,KAAK;AAAA,QAChB;AAAA,QACA,SAAS,CAAC,aAAa,WAAW;AAAA,QAClC;AAAA,MACF,CAAC;AAED,YAAM,WAAsD,CAAC;AAE7D,UAAI,QAAQ,WAAW;AACrB,iBAAS,IAAI,GAAG,IAAI,QAAQ,UAAU,QAAQ,KAAK;AACjD,mBAAS,KAAK;AAAA,YACZ,SAAS,QAAQ,UAAU,CAAC,KAAK;AAAA,YACjC,UAAU,QAAQ,YAAY,CAAC,KAAK,CAAC;AAAA,UACvC,CAAC;AAAA,QACH;AAAA,MACF;AAEA,aAAO;AAAA,IACT,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,+BAA+B,KAAK;AACtD,aAAO,CAAC;AAAA,IACV;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,gBAAgB,OAAsB;AAC5C,UAAM,QAAQ;AAAA,MACZ,UAAU,MAAM,KAAK;AAAA,MACrB,SAAS,MAAM,IAAI;AAAA,MACnB,WAAW,MAAM,MAAM;AAAA,IACzB;AAEA,QAAI,MAAM,aAAa;AACrB,YAAM,KAAK,gBAAgB,MAAM,WAAW,EAAE;AAAA,IAChD;AAEA,QAAI,MAAM,UAAU,MAAM,OAAO,SAAS,GAAG;AAC3C,YAAM,KAAK,WAAW,MAAM,OAAO,KAAK,IAAI,CAAC,EAAE;AAAA,IACjD;AAEA,QAAI,MAAM,WAAW,MAAM,QAAQ,SAAS,GAAG;AAC7C,YAAM,KAAK,YAAY,MAAM,QAAQ,KAAK,IAAI,CAAC,EAAE;AAAA,IACnD;AAEA,QAAI,MAAM,QAAQ,MAAM,KAAK,SAAS,GAAG;AACvC,YAAM,KAAK,SAAS,MAAM,KAAK,KAAK,IAAI,CAAC,EAAE;AAAA,IAC7C;AAEA,QAAI,MAAM,aAAa;AACrB,UAAI;AACF,cAAM,SAAS,KAAK,MAAM,MAAM,WAAW;AAC3C,YAAI,OAAO,SAAS;AAClB,gBAAM,KAAK,YAAY,OAAO,OAAO,EAAE;AAAA,QACzC;AACA,YAAI,OAAO,cAAc;AACvB,gBAAM,KAAK,cAAc,OAAO,aAAa,KAAK,IAAI,CAAC,EAAE;AAAA,QAC3D;AAAA,MACF,QAAQ;AAAA,MAER;AAAA,IACF;AAEA,WAAO,MAAM,KAAK,IAAI;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,aAAa,WAAkC;AACnD,SAAK,SAAS;AACd,SAAK,OAAO,KAAK,sBAAsB,SAAS,aAAa,KAAK,MAAM,EAAE;AAAA,EAC5E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,WAKH;AACD,QAAI,CAAC,KAAK,YAAY;AACpB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,QAAI;AAEF,YAAM,cAAc,MAAM,KAAK,WAAW,IAAI;AAAA,QAC5C,OAAO,EAAE,SAAS,KAAK,OAAO;AAAA,QAC9B,SAAS,CAAC,WAAW;AAAA,MACvB,CAAC;AAED,YAAM,QAAa;AAAA,QACjB,gBAAgB;AAAA,QAChB,eAAe,YAAY,KAAK,UAAU;AAAA,QAC1C,iBAAiB,CAAC;AAAA,MACpB;AAGA,UAAI,YAAY,WAAW;AACzB,mBAAW,YAAY,YAAY,WAAW;AAC5C,gBAAM,OAAO,UAAU,QAAQ;AAC/B,gBAAM,gBAAgB,IAAI,KAAK,MAAM,gBAAgB,IAAI,KAAK,KAAK;AAAA,QACrE;AAAA,MACF;AAGA,UAAI,KAAK,QAAQ;AACf,cAAM,cAAc,MAAM,KAAK,WAAW,IAAI;AAAA,UAC5C,OAAO,EAAE,SAAS,KAAK,OAAO;AAAA,UAC9B,SAAS,CAAC,KAAK;AAAA,QACjB,CAAC;AACD,cAAM,gBAAgB,YAAY,KAAK,UAAU;AAAA,MACnD;AAEA,YAAM,iBAAiB,MAAM,iBAAiB,MAAM,iBAAiB;AAErE,aAAO;AAAA,IACT,SAAS,OAAO;AACd,WAAK,OAAO,MAAM,uBAAuB,KAAK;AAC9C,YAAM;AAAA,IACR;AAAA,EACF;AACF;",
+  "names": []
+}

package/dist/core/storage/railway-optimized-storage.js

@@ -0,0 +1,550 @@
+import { createClient } from "redis";
+import { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from "@aws-sdk/client-s3";
+import { Storage } from "@google-cloud/storage";
+import { logger } from "../monitoring/logger.js";
+import * as zlib from "zlib";
+import { promisify } from "util";
+const gzipAsync = promisify(zlib.gzip);
+const gunzipAsync = promisify(zlib.gunzip);
+var StorageTier = /* @__PURE__ */ ((StorageTier2) => {
+  StorageTier2["HOT"] = "hot";
+  StorageTier2["WARM"] = "warm";
+  StorageTier2["COLD"] = "cold";
+  return StorageTier2;
+})(StorageTier || {});
+const DEFAULT_RAILWAY_CONFIG = {
+  redis: {
+    url: process.env.REDIS_URL || "redis://localhost:6379",
+    ttlSeconds: 86400,
+    // 24 hours
+    maxMemoryMb: 100
+    // 100MB Redis limit
+  },
+  railwayBuckets: {
+    endpoint: process.env.RAILWAY_BUCKET_ENDPOINT || "https://buckets.railway.app",
+    bucket: process.env.RAILWAY_BUCKET_NAME || "stackmemory-warm",
+    accessKeyId: process.env.RAILWAY_BUCKET_ACCESS_KEY || "",
+    secretAccessKey: process.env.RAILWAY_BUCKET_SECRET_KEY || "",
+    region: "us-east-1"
+  },
+  gcs: {
+    bucketName: process.env.GCS_BUCKET || "stackmemory-cold",
+    projectId: process.env.GCP_PROJECT_ID || "stackmemory",
+    keyFilename: process.env.GCP_KEY_FILE
+  },
+  tiers: {
+    hotHours: 24,
+    warmDays: 30,
+    compressionScore: 0.4
+  }
+};
+class RailwayOptimizedStorage {
+  redisClient;
+  railwayS3;
+  gcsStorage;
+  localDb;
+  config;
+  configManager;
+  metricsCache = /* @__PURE__ */ new Map();
+  initialized;
+  constructor(localDb, configManager, config) {
+    this.localDb = localDb;
+    this.configManager = configManager;
+    this.config = { ...DEFAULT_RAILWAY_CONFIG, ...config };
+    this.initializeSchema();
+    this.initialized = this.initializeClients();
+  }
+  /**
+   * Initialize storage clients
+   */
+  async initializeClients() {
+    if (this.config.redis.url) {
+      try {
+        this.redisClient = createClient({ url: this.config.redis.url });
+        this.redisClient.on("error", (err) => {
+          logger.error("Redis client error", err);
+        });
+        await this.redisClient.connect();
+        await this.redisClient.configSet("maxmemory-policy", "allkeys-lru");
+        logger.info("Redis connected for hot tier storage");
+      } catch (error) {
+        logger.warn("Redis connection failed, falling back to SQLite only", error);
+      }
+    }
+    if (this.config.railwayBuckets.accessKeyId) {
+      this.railwayS3 = new S3Client({
+        endpoint: this.config.railwayBuckets.endpoint,
+        region: this.config.railwayBuckets.region,
+        credentials: {
+          accessKeyId: this.config.railwayBuckets.accessKeyId,
+          secretAccessKey: this.config.railwayBuckets.secretAccessKey
+        },
+        forcePathStyle: true
+        // Required for Railway buckets
+      });
+      logger.info("Railway Buckets configured for warm tier");
+    }
+    if (this.config.gcs.projectId) {
+      try {
+        this.gcsStorage = new Storage({
+          projectId: this.config.gcs.projectId,
+          keyFilename: this.config.gcs.keyFilename
+        });
+        logger.info("GCS configured for cold tier storage");
+      } catch (error) {
+        logger.warn("GCS setup failed, will use Railway buckets only", error);
+      }
+    }
+  }
+  /**
+   * Initialize database schema for tracking
+   */
+  initializeSchema() {
+    this.localDb.exec(`
+      CREATE TABLE IF NOT EXISTS storage_tiers (
+        trace_id TEXT PRIMARY KEY,
+        tier TEXT NOT NULL,
+        location TEXT NOT NULL,
+        original_size INTEGER,
+        compressed_size INTEGER,
+        compression_ratio REAL,
+        access_count INTEGER DEFAULT 0,
+        last_accessed INTEGER,
+        created_at INTEGER,
+        migrated_at INTEGER,
+        score REAL,
+        FOREIGN KEY (trace_id) REFERENCES traces(id) ON DELETE CASCADE
+      )
+    `);
+    this.localDb.exec(`
+      CREATE INDEX IF NOT EXISTS idx_storage_tier ON storage_tiers(tier);
+      CREATE INDEX IF NOT EXISTS idx_storage_created ON storage_tiers(created_at);
+      CREATE INDEX IF NOT EXISTS idx_storage_accessed ON storage_tiers(last_accessed);
+    `);
+  }
+  /**
+   * Store a trace in the appropriate tier
+   */
+  async storeTrace(trace) {
+    await this.initialized;
+    const score = trace.score;
+    const age = Date.now() - trace.metadata.startTime;
+    const ageHours = age / (1e3 * 60 * 60);
+    let tier;
+    if (ageHours < this.config.tiers.hotHours && score > this.config.tiers.compressionScore) {
+      tier = "hot" /* HOT */;
+    } else if (ageHours < this.config.tiers.warmDays * 24) {
+      tier = "warm" /* WARM */;
+    } else {
+      tier = "cold" /* COLD */;
+    }
+    switch (tier) {
+      case "hot" /* HOT */:
+        await this.storeInRedis(trace);
+        break;
+      case "warm" /* WARM */:
+        await this.storeInRailwayBuckets(trace);
+        break;
+      case "cold" /* COLD */:
+        await this.storeInGCS(trace);
+        break;
+    }
+    this.trackStorage(trace.id, tier, trace);
+    return tier;
+  }
+  /**
+   * Store trace in Redis (hot tier)
+   */
+  async storeInRedis(trace) {
+    if (!this.redisClient) {
+      return;
+    }
+    try {
+      const key = `trace:${trace.id}`;
+      const data = JSON.stringify(trace);
+      let storedData;
+      if (data.length > 1e4) {
+        const compressed = await gzipAsync(data);
+        storedData = compressed.toString("base64");
+        await this.redisClient.hSet(key, {
+          data: storedData,
+          compressed: "true",
+          score: trace.score.toString(),
+          type: trace.type,
+          timestamp: trace.metadata.startTime.toString()
+        });
+      } else {
+        storedData = data;
+        await this.redisClient.hSet(key, {
+          data: storedData,
+          compressed: "false",
+          score: trace.score.toString(),
+          type: trace.type,
+          timestamp: trace.metadata.startTime.toString()
+        });
+      }
+      await this.redisClient.expire(key, this.config.redis.ttlSeconds);
+      await this.redisClient.zAdd("traces:by_score", {
+        score: trace.score,
+        value: trace.id
+      });
+      await this.redisClient.zAdd("traces:by_time", {
+        score: trace.metadata.startTime,
+        value: trace.id
+      });
+      logger.debug("Stored trace in Redis", {
+        traceId: trace.id,
+        size: data.length,
+        compressed: data.length > 1e4
+      });
+    } catch (error) {
+      logger.error("Failed to store in Redis", error);
+      throw error;
+    }
+  }
+  /**
+   * Store trace in Railway Buckets (warm tier)
+   */
+  async storeInRailwayBuckets(trace) {
+    if (!this.railwayS3) {
+      throw new Error("Railway Buckets not configured");
+    }
+    try {
+      const data = JSON.stringify(trace);
+      const compressed = await gzipAsync(data);
+      const date = new Date(trace.metadata.startTime);
+      const key = `traces/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${String(date.getDate()).padStart(2, "0")}/${trace.id}.json.gz`;
+      const command = new PutObjectCommand({
+        Bucket: this.config.railwayBuckets.bucket,
+        Key: key,
+        Body: compressed,
+        ContentType: "application/gzip",
+        Metadata: {
+          "trace-id": trace.id,
+          "trace-type": trace.type,
+          "trace-score": trace.score.toString(),
+          "original-size": data.length.toString(),
+          "compressed-size": compressed.length.toString()
+        }
+      });
+      await this.railwayS3.send(command);
+      if (this.redisClient) {
+        await this.redisClient.del(`trace:${trace.id}`);
+      }
+      logger.info("Stored trace in Railway Buckets", {
+        traceId: trace.id,
+        key,
+        originalSize: data.length,
+        compressedSize: compressed.length,
+        compressionRatio: (1 - compressed.length / data.length).toFixed(2)
+      });
+    } catch (error) {
+      logger.error("Failed to store in Railway Buckets", error);
+      throw error;
+    }
+  }
+  /**
+   * Store trace in GCS (cold tier)
+   */
+  async storeInGCS(trace) {
+    if (!this.gcsStorage) {
+      return this.storeInRailwayBuckets(trace);
+    }
+    try {
+      const minimal = this.createMinimalTrace(trace);
+      const data = JSON.stringify(minimal);
+      const compressed = await gzipAsync(data);
+      const date = new Date(trace.metadata.startTime);
+      const key = `archive/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${trace.id}.json.gz`;
+      const bucket = this.gcsStorage.bucket(this.config.gcs.bucketName);
+      const file = bucket.file(key);
+      await file.save(compressed, {
+        metadata: {
+          contentType: "application/gzip",
+          metadata: {
+            traceId: trace.id,
+            traceType: trace.type,
+            score: trace.score.toString(),
+            originalTools: trace.tools.length.toString()
+          }
+        },
+        storageClass: "COLDLINE"
+        // Use Coldline for cost optimization
+      });
+      if (this.railwayS3) {
+        try {
+          const warmKey = this.getWarmTierKey(trace);
+          await this.railwayS3.send(new DeleteObjectCommand({
+            Bucket: this.config.railwayBuckets.bucket,
+            Key: warmKey
+          }));
+        } catch (error) {
+        }
+      }
+      logger.info("Archived trace to GCS", {
+        traceId: trace.id,
+        key,
+        originalSize: JSON.stringify(trace).length,
+        compressedSize: compressed.length
+      });
+    } catch (error) {
+      logger.error("Failed to store in GCS", error);
+      throw error;
+    }
+  }
+  /**
+   * Create minimal trace for cold storage
+   */
+  createMinimalTrace(trace) {
+    return {
+      id: trace.id,
+      type: trace.type,
+      score: trace.score,
+      summary: trace.summary,
+      metadata: {
+        startTime: trace.metadata.startTime,
+        endTime: trace.metadata.endTime,
+        filesModified: trace.metadata.filesModified.length,
+        errorsCount: trace.metadata.errorsEncountered.length,
+        decisionsCount: trace.metadata.decisionsRecorded.length,
+        causalChain: trace.metadata.causalChain
+      },
+      toolSummary: {
+        count: trace.tools.length,
+        types: [...new Set(trace.tools.map((t) => t.tool))],
+        firstTool: trace.tools[0]?.tool,
+        lastTool: trace.tools[trace.tools.length - 1]?.tool
+      },
+      compressed: trace.compressed
+    };
+  }
+  /**
+   * Retrieve a trace from any tier
+   */
+  async retrieveTrace(traceId) {
+    await this.initialized;
+    const location = this.localDb.prepare(
+      "SELECT tier, location FROM storage_tiers WHERE trace_id = ?"
+    ).get(traceId);
+    if (!location) {
+      return null;
+    }
+    this.localDb.prepare(
+      "UPDATE storage_tiers SET access_count = access_count + 1, last_accessed = ? WHERE trace_id = ?"
+    ).run(Date.now(), traceId);
+    switch (location.tier) {
+      case "hot" /* HOT */:
+        return this.retrieveFromRedis(traceId);
+      case "warm" /* WARM */:
+        return this.retrieveFromRailwayBuckets(traceId, location.location);
+      case "cold" /* COLD */:
+        return this.retrieveFromGCS(traceId, location.location);
+      default:
+        return null;
+    }
+  }
+  /**
+   * Retrieve from Redis
+   */
+  async retrieveFromRedis(traceId) {
+    if (!this.redisClient) return null;
+    try {
+      const key = `trace:${traceId}`;
+      const data = await this.redisClient.hGetAll(key);
+      if (!data || !data.data) return null;
+      let traceData;
+      if (data.compressed === "true") {
+        const compressed = Buffer.from(data.data, "base64");
+        const decompressed = await gunzipAsync(compressed);
+        traceData = decompressed.toString();
+      } else {
+        traceData = data.data;
+      }
+      return JSON.parse(traceData);
+    } catch (error) {
+      logger.error("Failed to retrieve from Redis", error);
+      return null;
+    }
+  }
+  /**
+   * Retrieve from Railway Buckets
+   */
+  async retrieveFromRailwayBuckets(traceId, key) {
+    if (!this.railwayS3) return null;
+    try {
+      const command = new GetObjectCommand({
+        Bucket: this.config.railwayBuckets.bucket,
+        Key: key
+      });
+      const response = await this.railwayS3.send(command);
+      const compressed = await response.Body?.transformToByteArray();
+      if (!compressed) return null;
+      const decompressed = await gunzipAsync(Buffer.from(compressed));
+      return JSON.parse(decompressed.toString());
+    } catch (error) {
+      logger.error("Failed to retrieve from Railway Buckets", error);
+      return null;
+    }
+  }
+  /**
+   * Retrieve from GCS
+   */
+  async retrieveFromGCS(traceId, key) {
+    if (!this.gcsStorage) return null;
+    try {
+      const bucket = this.gcsStorage.bucket(this.config.gcs.bucketName);
+      const file = bucket.file(key);
+      const [compressed] = await file.download();
+      const decompressed = await gunzipAsync(compressed);
+      return JSON.parse(decompressed.toString());
+    } catch (error) {
+      logger.error("Failed to retrieve from GCS", error);
+      return null;
+    }
+  }
+  /**
+   * Track storage in database
+   */
+  trackStorage(traceId, tier, trace) {
+    const originalSize = JSON.stringify(trace).length;
+    const compressedSize = Math.floor(originalSize * 0.3);
+    this.localDb.prepare(`
+      INSERT OR REPLACE INTO storage_tiers (
+        trace_id, tier, location, original_size, compressed_size,
+        compression_ratio, access_count, last_accessed, created_at,
+        migrated_at, score
+      ) VALUES (?, ?, ?, ?, ?, ?, 0, ?, ?, ?, ?)
+    `).run(
+      traceId,
+      tier,
+      this.getStorageLocation(trace, tier),
+      originalSize,
+      compressedSize,
+      1 - compressedSize / originalSize,
+      Date.now(),
+      trace.metadata.startTime,
+      Date.now(),
+      trace.score
+    );
+  }
+  /**
+   * Get storage location key
+   */
+  getStorageLocation(trace, tier) {
+    const date = new Date(trace.metadata.startTime);
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, "0");
+    const day = String(date.getDate()).padStart(2, "0");
+    switch (tier) {
+      case "hot" /* HOT */:
+        return `redis:trace:${trace.id}`;
+      case "warm" /* WARM */:
+        return `traces/${year}/${month}/${day}/${trace.id}.json.gz`;
+      case "cold" /* COLD */:
+        return `archive/${year}/${month}/${trace.id}.json.gz`;
+    }
+  }
+  /**
+   * Get warm tier key for a trace
+   */
+  getWarmTierKey(trace) {
+    const date = new Date(trace.metadata.startTime);
+    return `traces/${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}/${String(date.getDate()).padStart(2, "0")}/${trace.id}.json.gz`;
+  }
+  /**
+   * Migrate traces between tiers based on age
+   */
+  async migrateTiers() {
+    const results = {
+      hotToWarm: 0,
+      warmToCold: 0,
+      errors: []
+    };
+    const now = Date.now();
+    const candidates = this.localDb.prepare(`
+      SELECT trace_id, tier, created_at, score
+      FROM storage_tiers
+      WHERE tier != 'cold'
+      ORDER BY created_at ASC
+    `).all();
+    for (const candidate of candidates) {
+      const ageHours = (now - candidate.created_at) / (1e3 * 60 * 60);
+      try {
+        if (candidate.tier === "hot" /* HOT */ && ageHours > this.config.tiers.hotHours) {
+          const trace = await this.retrieveTrace(candidate.trace_id);
+          if (trace) {
+            await this.storeInRailwayBuckets(trace);
+            this.trackStorage(candidate.trace_id, "warm" /* WARM */, trace);
+            results.hotToWarm++;
+          }
+        } else if (candidate.tier === "warm" /* WARM */ && ageHours > this.config.tiers.warmDays * 24) {
+          const trace = await this.retrieveTrace(candidate.trace_id);
+          if (trace) {
+            await this.storeInGCS(trace);
+            this.trackStorage(candidate.trace_id, "cold" /* COLD */, trace);
+            results.warmToCold++;
+          }
+        }
+      } catch (error) {
+        results.errors.push(`Failed to migrate ${candidate.trace_id}: ${error}`);
+      }
+    }
+    logger.info("Tier migration completed", results);
+    return results;
+  }
+  /**
+   * Get storage statistics
+   */
+  getStorageStats() {
+    const tierStats = this.localDb.prepare(`
+      SELECT 
+        tier,
+        COUNT(*) as count,
+        SUM(original_size) as total_original,
+        SUM(compressed_size) as total_compressed,
+        AVG(compression_ratio) as avg_compression,
+        AVG(access_count) as avg_access
+      FROM storage_tiers
+      GROUP BY tier
+    `).all();
+    const ageDistribution = this.localDb.prepare(`
+      SELECT 
+        CASE 
+          WHEN (? - created_at) / 3600000 < 24 THEN '< 24h'
+          WHEN (? - created_at) / 86400000 < 7 THEN '1-7d'
+          WHEN (? - created_at) / 86400000 < 30 THEN '7-30d'
+          ELSE '30d+'
+        END as age_group,
+        COUNT(*) as count
+      FROM storage_tiers
+      GROUP BY age_group
+    `).all(Date.now(), Date.now(), Date.now());
+    return {
+      byTier: tierStats,
+      byAge: ageDistribution,
+      totalTraces: tierStats.reduce((sum, t) => sum + t.count, 0),
+      totalSize: tierStats.reduce((sum, t) => sum + t.total_original, 0),
+      compressedSize: tierStats.reduce((sum, t) => sum + t.total_compressed, 0)
+    };
+  }
+  /**
+   * Clean up expired data
+   */
+  async cleanup() {
+    let cleaned = 0;
+    const cutoff = Date.now() - 90 * 24 * 60 * 60 * 1e3;
+    const result = this.localDb.prepare(`
+      DELETE FROM storage_tiers
+      WHERE tier = 'cold' AND created_at < ? AND access_count = 0
+    `).run(cutoff);
+    cleaned = result.changes;
+    logger.info("Cleanup completed", { removed: cleaned });
+    return cleaned;
+  }
+}
+export {
+  DEFAULT_RAILWAY_CONFIG,
+  RailwayOptimizedStorage,
+  StorageTier
+};
+//# sourceMappingURL=railway-optimized-storage.js.map