@stackmemoryai/stackmemory 0.3.0 → 0.3.3

package/dist/core/storage/remote-storage.js

@@ -0,0 +1,456 @@
+import { S3Client, PutObjectCommand, GetObjectCommand } from "@aws-sdk/client-s3";
+import { Storage } from "@google-cloud/storage";
+import { logger } from "../monitoring/logger.js";
+var StorageTier = /* @__PURE__ */ ((StorageTier2) => {
+  StorageTier2["HOT"] = "hot";
+  StorageTier2["NEARLINE"] = "nearline";
+  StorageTier2["COLDLINE"] = "coldline";
+  StorageTier2["ARCHIVE"] = "archive";
+  return StorageTier2;
+})(StorageTier || {});
+const DEFAULT_REMOTE_CONFIG = {
+  provider: "gcs",
+  // Default to GCS for better pricing
+  gcs: {
+    bucketName: "stackmemory-traces",
+    projectId: process.env.GCP_PROJECT_ID || "stackmemory"
+  },
+  timeseries: {
+    type: "sqlite",
+    // Use SQLite for development
+    host: "localhost",
+    port: 0,
+    database: "stackmemory_timeseries"
+  },
+  migration: {
+    batchSize: 100,
+    hotAgeHours: 168,
+    // 7 days
+    nearlineAgeHours: 720,
+    // 30 days
+    coldlineAgeHours: 2160,
+    // 90 days
+    archiveAgeHours: 8760,
+    // 365 days
+    scoreThreshold: 0.4
+  }
+};
+class RemoteStorageManager {
+  storageClient;
+  config;
+  localDb;
+  migrationInProgress = false;
+  constructor(localDb, config) {
+    this.localDb = localDb;
+    this.config = { ...DEFAULT_REMOTE_CONFIG, ...config };
+    this.initializeStorageClient();
+    this.initializeSchema();
+  }
+  /**
+   * Initialize storage client based on provider
+   */
+  initializeStorageClient() {
+    switch (this.config.provider) {
+      case "gcs":
+        if (this.config.gcs) {
+          this.storageClient = new Storage({
+            projectId: this.config.gcs.projectId,
+            keyFilename: this.config.gcs.keyFilename
+          });
+        }
+        break;
+      case "s3":
+      case "railway":
+        if (this.config.s3?.accessKeyId && this.config.s3?.secretAccessKey) {
+          this.storageClient = new S3Client({
+            region: this.config.s3.region,
+            credentials: {
+              accessKeyId: this.config.s3.accessKeyId,
+              secretAccessKey: this.config.s3.secretAccessKey
+            },
+            endpoint: this.config.s3.endpoint
+            // Railway buckets endpoint
+          });
+        }
+        break;
+    }
+  }
+  /**
+   * Initialize migration tracking schema
+   */
+  initializeSchema() {
+    this.localDb.exec(`
+      CREATE TABLE IF NOT EXISTS remote_migrations (
+        trace_id TEXT PRIMARY KEY,
+        migrated_at INTEGER NOT NULL,
+        storage_tier TEXT NOT NULL,
+        s3_key TEXT,
+        timeseries_id TEXT,
+        compression_level TEXT,
+        original_size INTEGER,
+        compressed_size INTEGER,
+        retrieval_count INTEGER DEFAULT 0,
+        last_retrieved INTEGER,
+        FOREIGN KEY (trace_id) REFERENCES traces(id) ON DELETE CASCADE
+      )
+    `);
+    this.localDb.exec(`
+      CREATE INDEX IF NOT EXISTS idx_migrations_tier ON remote_migrations(storage_tier);
+      CREATE INDEX IF NOT EXISTS idx_migrations_migrated ON remote_migrations(migrated_at);
+    `);
+  }
+  /**
+   * Identify traces for migration based on age and importance
+   */
+  async identifyMigrationCandidates() {
+    const now = Date.now();
+    const traces = this.localDb.prepare(`
+      SELECT 
+        t.id,
+        t.score,
+        t.start_time,
+        LENGTH(t.compressed_data) + 
+        COALESCE((SELECT SUM(LENGTH(tc.arguments) + LENGTH(tc.result)) 
+                  FROM tool_calls tc WHERE tc.trace_id = t.id), 0) as size,
+        rm.trace_id as already_migrated
+      FROM traces t
+      LEFT JOIN remote_migrations rm ON t.id = rm.trace_id
+      WHERE rm.trace_id IS NULL  -- Not already migrated
+      ORDER BY t.start_time ASC
+    `).all();
+    const candidates = [];
+    for (const trace of traces) {
+      const ageHours = (now - trace.start_time) / (1e3 * 60 * 60);
+      const candidate = this.evaluateTrace(
+        trace.id,
+        ageHours,
+        trace.score,
+        trace.size || 0
+      );
+      candidates.push(candidate);
+    }
+    return candidates;
+  }
+  /**
+   * Evaluate a trace for migration based on GCS storage classes
+   */
+  evaluateTrace(traceId, ageHours, score, size) {
+    let tier = "hot" /* HOT */;
+    let shouldMigrate = false;
+    let compressionLevel = "none";
+    if (ageHours > this.config.migration.archiveAgeHours) {
+      tier = "archive" /* ARCHIVE */;
+      shouldMigrate = true;
+      compressionLevel = "heavy";
+    } else if (ageHours > this.config.migration.coldlineAgeHours) {
+      tier = "coldline" /* COLDLINE */;
+      shouldMigrate = true;
+      compressionLevel = "heavy";
+    } else if (ageHours > this.config.migration.nearlineAgeHours) {
+      tier = "nearline" /* NEARLINE */;
+      shouldMigrate = true;
+      compressionLevel = "medium";
+    } else if (ageHours > this.config.migration.hotAgeHours) {
+      tier = "hot" /* HOT */;
+      if (score < this.config.migration.scoreThreshold) {
+        shouldMigrate = true;
+        compressionLevel = "light";
+      }
+    }
+    const localSizeLimit = 2 * 1024 * 1024 * 1024;
+    const currentLocalSize = this.getLocalStorageSize();
+    if (currentLocalSize > localSizeLimit * 0.75) {
+      shouldMigrate = true;
+      if (compressionLevel === "none") {
+        compressionLevel = "light";
+      }
+    }
+    return {
+      traceId,
+      age: ageHours,
+      score,
+      size,
+      tier,
+      shouldMigrate,
+      compressionLevel
+    };
+  }
+  /**
+   * Migrate traces to remote storage
+   */
+  async migrateTraces(candidates, dryRun = false) {
+    if (this.migrationInProgress) {
+      return {
+        migrated: 0,
+        failed: 0,
+        totalSize: 0,
+        errors: ["Migration already in progress"]
+      };
+    }
+    this.migrationInProgress = true;
+    const results = {
+      migrated: 0,
+      failed: 0,
+      totalSize: 0,
+      errors: []
+    };
+    try {
+      const toMigrate = candidates.filter((c) => c.shouldMigrate);
+      const batches = this.createBatches(toMigrate, this.config.migration.batchSize);
+      for (const batch of batches) {
+        if (dryRun) {
+          logger.info("Dry run - would migrate batch", {
+            count: batch.length,
+            totalSize: batch.reduce((sum, c) => sum + c.size, 0)
+          });
+          results.migrated += batch.length;
+          continue;
+        }
+        const batchResults = await this.migrateBatch(batch);
+        results.migrated += batchResults.success;
+        results.failed += batchResults.failed;
+        results.totalSize += batchResults.totalSize;
+        results.errors.push(...batchResults.errors);
+        await new Promise((resolve) => setTimeout(resolve, 100));
+      }
+    } finally {
+      this.migrationInProgress = false;
+    }
+    logger.info("Migration completed", results);
+    return results;
+  }
+  /**
+   * Migrate a batch of traces
+   */
+  async migrateBatch(batch) {
+    const results = {
+      success: 0,
+      failed: 0,
+      totalSize: 0,
+      errors: []
+    };
+    for (const candidate of batch) {
+      try {
+        const trace = this.getTraceData(candidate.traceId);
+        if (!trace) {
+          throw new Error(`Trace ${candidate.traceId} not found`);
+        }
+        const compressed = this.compressTrace(trace, candidate.compressionLevel);
+        if (this.s3Client) {
+          const s3Key = this.generateS3Key(candidate);
+          await this.uploadToS3(s3Key, compressed);
+          this.recordMigration(candidate, s3Key, trace, compressed);
+        } else {
+          this.recordMigration(candidate, "simulated", trace, compressed);
+        }
+        if (candidate.tier === StorageTier.COLD || candidate.tier === "archive" /* ARCHIVE */) {
+          this.removeLocalTrace(candidate.traceId);
+        }
+        results.success++;
+        results.totalSize += candidate.size;
+      } catch (error) {
+        results.failed++;
+        results.errors.push(
+          `Failed to migrate ${candidate.traceId}: ${error}`
+        );
+        logger.error("Migration failed for trace", {
+          traceId: candidate.traceId,
+          error
+        });
+      }
+    }
+    return results;
+  }
+  /**
+   * Get full trace data for migration
+   */
+  getTraceData(traceId) {
+    const traceRow = this.localDb.prepare(
+      "SELECT * FROM traces WHERE id = ?"
+    ).get(traceId);
+    if (!traceRow) return null;
+    const toolCalls = this.localDb.prepare(
+      "SELECT * FROM tool_calls WHERE trace_id = ? ORDER BY sequence_number"
+    ).all(traceId);
+    return {
+      trace: traceRow,
+      toolCalls
+    };
+  }
+  /**
+   * Compress trace based on compression level
+   */
+  compressTrace(data, level) {
+    let jsonData = JSON.stringify(data);
+    switch (level) {
+      case "none":
+        return Buffer.from(jsonData);
+      case "light":
+        return Buffer.from(JSON.stringify(JSON.parse(jsonData)));
+      case "medium":
+        const cleaned = JSON.parse(
+          jsonData,
+          (key, value) => value === null || value === void 0 ? void 0 : value
+        );
+        return Buffer.from(JSON.stringify(cleaned));
+      case "heavy":
+        const minimal = {
+          id: data.trace.id,
+          type: data.trace.type,
+          score: data.trace.score,
+          summary: data.trace.summary,
+          timestamps: {
+            start: data.trace.start_time,
+            end: data.trace.end_time
+          },
+          toolCount: data.toolCalls.length,
+          toolTypes: [...new Set(data.toolCalls.map((t) => t.tool))]
+        };
+        return Buffer.from(JSON.stringify(minimal));
+      default:
+        return Buffer.from(jsonData);
+    }
+  }
+  /**
+   * Generate S3 key for trace
+   */
+  generateS3Key(candidate) {
+    const date = new Date(Date.now() - candidate.age * 60 * 60 * 1e3);
+    const year = date.getFullYear();
+    const month = String(date.getMonth() + 1).padStart(2, "0");
+    const day = String(date.getDate()).padStart(2, "0");
+    return `traces/${year}/${month}/${day}/${candidate.tier}/${candidate.traceId}.json`;
+  }
+  /**
+   * Upload to S3
+   */
+  async uploadToS3(key, data) {
+    if (!this.s3Client) {
+      throw new Error("S3 client not configured");
+    }
+    const command = new PutObjectCommand({
+      Bucket: this.config.s3.bucket,
+      Key: key,
+      Body: data,
+      ContentType: "application/json",
+      Metadata: {
+        "trace-version": "1.0",
+        "compression": "true"
+      }
+    });
+    await this.s3Client.send(command);
+  }
+  /**
+   * Record migration in local database
+   */
+  recordMigration(candidate, s3Key, originalData, compressedData) {
+    const stmt = this.localDb.prepare(`
+      INSERT INTO remote_migrations (
+        trace_id, migrated_at, storage_tier, s3_key,
+        compression_level, original_size, compressed_size
+      ) VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+    stmt.run(
+      candidate.traceId,
+      Date.now(),
+      candidate.tier,
+      s3Key,
+      candidate.compressionLevel,
+      JSON.stringify(originalData).length,
+      compressedData.length
+    );
+  }
+  /**
+   * Remove local trace after migration
+   */
+  removeLocalTrace(traceId) {
+    this.localDb.prepare("DELETE FROM tool_calls WHERE trace_id = ?").run(traceId);
+    this.localDb.prepare("DELETE FROM traces WHERE id = ?").run(traceId);
+  }
+  /**
+   * Get current local storage size
+   */
+  getLocalStorageSize() {
+    const result = this.localDb.prepare(`
+      SELECT 
+        SUM(LENGTH(compressed_data)) +
+        COALESCE((SELECT SUM(LENGTH(arguments) + LENGTH(result)) 
+                  FROM tool_calls), 0) as total_size
+      FROM traces
+    `).get();
+    return result?.total_size || 0;
+  }
+  /**
+   * Create batches from candidates
+   */
+  createBatches(items, batchSize) {
+    const batches = [];
+    for (let i = 0; i < items.length; i += batchSize) {
+      batches.push(items.slice(i, i + batchSize));
+    }
+    return batches;
+  }
+  /**
+   * Retrieve trace from remote storage
+   */
+  async retrieveTrace(traceId) {
+    const migration = this.localDb.prepare(`
+      SELECT * FROM remote_migrations WHERE trace_id = ?
+    `).get(traceId);
+    if (!migration) {
+      throw new Error(`Trace ${traceId} not found in remote storage`);
+    }
+    this.localDb.prepare(`
+      UPDATE remote_migrations 
+      SET retrieval_count = retrieval_count + 1, last_retrieved = ?
+      WHERE trace_id = ?
+    `).run(Date.now(), traceId);
+    if (!this.s3Client) {
+      throw new Error("S3 client not configured");
+    }
+    const command = new GetObjectCommand({
+      Bucket: this.config.s3.bucket,
+      Key: migration.s3_key
+    });
+    const response = await this.s3Client.send(command);
+    const data = await response.Body?.transformToString();
+    if (!data) {
+      throw new Error("No data retrieved from S3");
+    }
+    return JSON.parse(data);
+  }
+  /**
+   * Get migration statistics
+   */
+  getMigrationStats() {
+    const stats = this.localDb.prepare(`
+      SELECT 
+        storage_tier,
+        COUNT(*) as count,
+        SUM(original_size) as original_size,
+        SUM(compressed_size) as compressed_size,
+        AVG(retrieval_count) as avg_retrievals
+      FROM remote_migrations
+      GROUP BY storage_tier
+    `).all();
+    const total = this.localDb.prepare(`
+      SELECT 
+        COUNT(*) as total_migrated,
+        SUM(original_size) as total_original,
+        SUM(compressed_size) as total_compressed
+      FROM remote_migrations
+    `).get();
+    return {
+      byTier: stats,
+      total,
+      compressionRatio: total ? (1 - total.total_compressed / total.total_original).toFixed(2) : 0,
+      localSize: this.getLocalStorageSize()
+    };
+  }
+}
+export {
+  DEFAULT_REMOTE_CONFIG,
+  RemoteStorageManager,
+  StorageTier
+};
+//# sourceMappingURL=remote-storage.js.map

package/dist/core/storage/remote-storage.js.map

@@ -0,0 +1,7 @@
+{
+  "version": 3,
+  "sources": ["../../../src/core/storage/remote-storage.ts"],
+  "sourcesContent": ["/**\n * Remote Storage Interface for Two-Tier Storage System\n * Implements infinite retention with TimeSeries DB + S3\n */\n\nimport { S3Client, PutObjectCommand, GetObjectCommand, DeleteObjectCommand } from '@aws-sdk/client-s3';\nimport { Storage } from '@google-cloud/storage';\nimport { logger } from '../monitoring/logger.js';\nimport { Trace, CompressedTrace } from '../trace/types.js';\nimport Database from 'better-sqlite3';\n\nexport enum StorageTier {\n  HOT = 'hot',           // < 7 days - Railway Buckets or GCS Standard\n  NEARLINE = 'nearline', // 7-30 days - GCS Nearline ($0.01/GB)\n  COLDLINE = 'coldline', // 30-90 days - GCS Coldline ($0.004/GB)\n  ARCHIVE = 'archive'    // > 90 days - GCS Archive ($0.0012/GB)\n}\n\nexport interface RemoteStorageConfig {\n  provider: 'gcs' | 's3' | 'railway';\n  gcs?: {\n    bucketName: string;\n    projectId: string;\n    keyFilename?: string; // Path to service account key\n  };\n  s3?: {\n    bucket: string;\n    region: string;\n    accessKeyId?: string;\n    secretAccessKey?: string;\n    endpoint?: string; // For Railway buckets or MinIO\n  };\n  timeseries: {\n    type: 'clickhouse' | 'timescale' | 'influxdb' | 'sqlite'; // SQLite for dev\n    host: string;\n    port: number;\n    database: string;\n    username?: string;\n    password?: string;\n  };\n  migration: {\n    batchSize: number;\n    hotAgeHours: number;        // < 7 days\n    nearlineAgeHours: number;   // 7-30 days  \n    coldlineAgeHours: number;   // 30-90 days\n    archiveAgeHours: number;    // > 90 days\n    scoreThreshold: number;     // Score threshold for early migration\n  };\n}\n\nexport const DEFAULT_REMOTE_CONFIG: RemoteStorageConfig = {\n  provider: 'gcs',  // Default to GCS for better pricing\n  gcs: {\n    bucketName: 'stackmemory-traces',\n    projectId: process.env.GCP_PROJECT_ID || 'stackmemory',\n  },\n  timeseries: {\n    type: 'sqlite',  // Use SQLite for development\n    host: 'localhost',\n    port: 0,\n    database: 'stackmemory_timeseries',\n  },\n  migration: {\n    batchSize: 100,\n    hotAgeHours: 168,        // 7 days\n    nearlineAgeHours: 720,   // 30 days\n    coldlineAgeHours: 2160,  // 90 days\n    archiveAgeHours: 8760,   // 365 days\n    scoreThreshold: 0.4,\n  }\n};\n\nexport interface MigrationCandidate {\n  traceId: string;\n  age: number;\n  score: number;\n  size: number;\n  tier: StorageTier;\n  shouldMigrate: boolean;\n  compressionLevel: 'none' | 'light' | 'medium' | 'heavy';\n}\n\n/**\n * Remote storage manager for infinite trace retention\n */\nexport class RemoteStorageManager {\n  private storageClient?: S3Client | Storage;\n  private config: RemoteStorageConfig;\n  private localDb: Database.Database;\n  private migrationInProgress = false;\n  \n  constructor(\n    localDb: Database.Database,\n    config?: Partial<RemoteStorageConfig>\n  ) {\n    this.localDb = localDb;\n    this.config = { ...DEFAULT_REMOTE_CONFIG, ...config };\n    \n    this.initializeStorageClient();\n    this.initializeSchema();\n  }\n  \n  /**\n   * Initialize storage client based on provider\n   */\n  private initializeStorageClient(): void {\n    switch (this.config.provider) {\n      case 'gcs':\n        if (this.config.gcs) {\n          this.storageClient = new Storage({\n            projectId: this.config.gcs.projectId,\n            keyFilename: this.config.gcs.keyFilename,\n          });\n        }\n        break;\n        \n      case 's3':\n      case 'railway':\n        if (this.config.s3?.accessKeyId && this.config.s3?.secretAccessKey) {\n          this.storageClient = new S3Client({\n            region: this.config.s3.region,\n            credentials: {\n              accessKeyId: this.config.s3.accessKeyId,\n              secretAccessKey: this.config.s3.secretAccessKey,\n            },\n            endpoint: this.config.s3.endpoint, // Railway buckets endpoint\n          });\n        }\n        break;\n    }\n  }\n  \n  /**\n   * Initialize migration tracking schema\n   */\n  private initializeSchema(): void {\n    this.localDb.exec(`\n      CREATE TABLE IF NOT EXISTS remote_migrations (\n        trace_id TEXT PRIMARY KEY,\n        migrated_at INTEGER NOT NULL,\n        storage_tier TEXT NOT NULL,\n        s3_key TEXT,\n        timeseries_id TEXT,\n        compression_level TEXT,\n        original_size INTEGER,\n        compressed_size INTEGER,\n        retrieval_count INTEGER DEFAULT 0,\n        last_retrieved INTEGER,\n        FOREIGN KEY (trace_id) REFERENCES traces(id) ON DELETE CASCADE\n      )\n    `);\n    \n    this.localDb.exec(`\n      CREATE INDEX IF NOT EXISTS idx_migrations_tier ON remote_migrations(storage_tier);\n      CREATE INDEX IF NOT EXISTS idx_migrations_migrated ON remote_migrations(migrated_at);\n    `);\n  }\n  \n  /**\n   * Identify traces for migration based on age and importance\n   */\n  async identifyMigrationCandidates(): Promise<MigrationCandidate[]> {\n    const now = Date.now();\n    \n    // Query all traces with their metadata\n    const traces = this.localDb.prepare(`\n      SELECT \n        t.id,\n        t.score,\n        t.start_time,\n        LENGTH(t.compressed_data) + \n        COALESCE((SELECT SUM(LENGTH(tc.arguments) + LENGTH(tc.result)) \n                  FROM tool_calls tc WHERE tc.trace_id = t.id), 0) as size,\n        rm.trace_id as already_migrated\n      FROM traces t\n      LEFT JOIN remote_migrations rm ON t.id = rm.trace_id\n      WHERE rm.trace_id IS NULL  -- Not already migrated\n      ORDER BY t.start_time ASC\n    `).all() as any[];\n    \n    const candidates: MigrationCandidate[] = [];\n    \n    for (const trace of traces) {\n      const ageHours = (now - trace.start_time) / (1000 * 60 * 60);\n      const candidate = this.evaluateTrace(\n        trace.id,\n        ageHours,\n        trace.score,\n        trace.size || 0\n      );\n      \n      candidates.push(candidate);\n    }\n    \n    return candidates;\n  }\n  \n  /**\n   * Evaluate a trace for migration based on GCS storage classes\n   */\n  private evaluateTrace(\n    traceId: string,\n    ageHours: number,\n    score: number,\n    size: number\n  ): MigrationCandidate {\n    let tier = StorageTier.HOT;\n    let shouldMigrate = false;\n    let compressionLevel: 'none' | 'light' | 'medium' | 'heavy' = 'none';\n    \n    // Determine storage tier based on age and GCS storage classes\n    if (ageHours > this.config.migration.archiveAgeHours) {\n      // GCS Archive: $0.0012/GB - accessed < once per year\n      tier = StorageTier.ARCHIVE;\n      shouldMigrate = true;\n      compressionLevel = 'heavy';\n    } else if (ageHours > this.config.migration.coldlineAgeHours) {\n      // GCS Coldline: $0.004/GB - accessed < once per quarter  \n      tier = StorageTier.COLDLINE;\n      shouldMigrate = true;\n      compressionLevel = 'heavy';\n    } else if (ageHours > this.config.migration.nearlineAgeHours) {\n      // GCS Nearline: $0.01/GB - accessed < once per month\n      tier = StorageTier.NEARLINE;\n      shouldMigrate = true;\n      compressionLevel = 'medium';\n    } else if (ageHours > this.config.migration.hotAgeHours) {\n      // Still hot but consider migration if low importance\n      tier = StorageTier.HOT;\n      if (score < this.config.migration.scoreThreshold) {\n        shouldMigrate = true;\n        compressionLevel = 'light';\n      }\n    }\n    \n    // Force migration for size pressure\n    const localSizeLimit = 2 * 1024 * 1024 * 1024; // 2GB\n    const currentLocalSize = this.getLocalStorageSize();\n    \n    if (currentLocalSize > localSizeLimit * 0.75) {\n      // Start migrating when 75% full\n      shouldMigrate = true;\n      if (compressionLevel === 'none') {\n        compressionLevel = 'light';\n      }\n    }\n    \n    return {\n      traceId,\n      age: ageHours,\n      score,\n      size,\n      tier,\n      shouldMigrate,\n      compressionLevel,\n    };\n  }\n  \n  /**\n   * Migrate traces to remote storage\n   */\n  async migrateTraces(\n    candidates: MigrationCandidate[],\n    dryRun: boolean = false\n  ): Promise<{\n    migrated: number;\n    failed: number;\n    totalSize: number;\n    errors: string[];\n  }> {\n    if (this.migrationInProgress) {\n      return {\n        migrated: 0,\n        failed: 0,\n        totalSize: 0,\n        errors: ['Migration already in progress'],\n      };\n    }\n    \n    this.migrationInProgress = true;\n    const results = {\n      migrated: 0,\n      failed: 0,\n      totalSize: 0,\n      errors: [] as string[],\n    };\n    \n    try {\n      // Process in batches\n      const toMigrate = candidates.filter(c => c.shouldMigrate);\n      const batches = this.createBatches(toMigrate, this.config.migration.batchSize);\n      \n      for (const batch of batches) {\n        if (dryRun) {\n          logger.info('Dry run - would migrate batch', {\n            count: batch.length,\n            totalSize: batch.reduce((sum, c) => sum + c.size, 0),\n          });\n          results.migrated += batch.length;\n          continue;\n        }\n        \n        const batchResults = await this.migrateBatch(batch);\n        results.migrated += batchResults.success;\n        results.failed += batchResults.failed;\n        results.totalSize += batchResults.totalSize;\n        results.errors.push(...batchResults.errors);\n        \n        // Small delay between batches\n        await new Promise(resolve => setTimeout(resolve, 100));\n      }\n      \n    } finally {\n      this.migrationInProgress = false;\n    }\n    \n    logger.info('Migration completed', results);\n    return results;\n  }\n  \n  /**\n   * Migrate a batch of traces\n   */\n  private async migrateBatch(\n    batch: MigrationCandidate[]\n  ): Promise<{\n    success: number;\n    failed: number;\n    totalSize: number;\n    errors: string[];\n  }> {\n    const results = {\n      success: 0,\n      failed: 0,\n      totalSize: 0,\n      errors: [] as string[],\n    };\n    \n    for (const candidate of batch) {\n      try {\n        // Get full trace data\n        const trace = this.getTraceData(candidate.traceId);\n        if (!trace) {\n          throw new Error(`Trace ${candidate.traceId} not found`);\n        }\n        \n        // Compress based on level\n        const compressed = this.compressTrace(trace, candidate.compressionLevel);\n        \n        // Upload to S3\n        if (this.s3Client) {\n          const s3Key = this.generateS3Key(candidate);\n          await this.uploadToS3(s3Key, compressed);\n          \n          // Record migration\n          this.recordMigration(candidate, s3Key, trace, compressed);\n        } else {\n          // Local simulation for testing\n          this.recordMigration(candidate, 'simulated', trace, compressed);\n        }\n        \n        // Optionally remove from local after successful migration\n        if (candidate.tier === StorageTier.COLD || \n            candidate.tier === StorageTier.ARCHIVE) {\n          this.removeLocalTrace(candidate.traceId);\n        }\n        \n        results.success++;\n        results.totalSize += candidate.size;\n        \n      } catch (error) {\n        results.failed++;\n        results.errors.push(\n          `Failed to migrate ${candidate.traceId}: ${error}`\n        );\n        logger.error('Migration failed for trace', { \n          traceId: candidate.traceId, \n          error \n        });\n      }\n    }\n    \n    return results;\n  }\n  \n  /**\n   * Get full trace data for migration\n   */\n  private getTraceData(traceId: string): any {\n    const traceRow = this.localDb.prepare(\n      'SELECT * FROM traces WHERE id = ?'\n    ).get(traceId);\n    \n    if (!traceRow) return null;\n    \n    const toolCalls = this.localDb.prepare(\n      'SELECT * FROM tool_calls WHERE trace_id = ? ORDER BY sequence_number'\n    ).all(traceId);\n    \n    return {\n      trace: traceRow,\n      toolCalls,\n    };\n  }\n  \n  /**\n   * Compress trace based on compression level\n   */\n  private compressTrace(\n    data: any,\n    level: 'none' | 'light' | 'medium' | 'heavy'\n  ): Buffer {\n    let jsonData = JSON.stringify(data);\n    \n    // Apply different compression based on level\n    switch (level) {\n      case 'none':\n        return Buffer.from(jsonData);\n      \n      case 'light':\n        // Remove formatting, keep all data\n        return Buffer.from(JSON.stringify(JSON.parse(jsonData)));\n      \n      case 'medium':\n        // Remove null fields and compress\n        const cleaned = JSON.parse(jsonData, (key, value) => \n          value === null || value === undefined ? undefined : value\n        );\n        return Buffer.from(JSON.stringify(cleaned));\n      \n      case 'heavy':\n        // Remove tool results and arguments, keep only essential\n        const minimal = {\n          id: data.trace.id,\n          type: data.trace.type,\n          score: data.trace.score,\n          summary: data.trace.summary,\n          timestamps: {\n            start: data.trace.start_time,\n            end: data.trace.end_time,\n          },\n          toolCount: data.toolCalls.length,\n          toolTypes: [...new Set(data.toolCalls.map((t: any) => t.tool))],\n        };\n        return Buffer.from(JSON.stringify(minimal));\n      \n      default:\n        return Buffer.from(jsonData);\n    }\n  }\n  \n  /**\n   * Generate S3 key for trace\n   */\n  private generateS3Key(candidate: MigrationCandidate): string {\n    const date = new Date(Date.now() - candidate.age * 60 * 60 * 1000);\n    const year = date.getFullYear();\n    const month = String(date.getMonth() + 1).padStart(2, '0');\n    const day = String(date.getDate()).padStart(2, '0');\n    \n    return `traces/${year}/${month}/${day}/${candidate.tier}/${candidate.traceId}.json`;\n  }\n  \n  /**\n   * Upload to S3\n   */\n  private async uploadToS3(key: string, data: Buffer): Promise<void> {\n    if (!this.s3Client) {\n      throw new Error('S3 client not configured');\n    }\n    \n    const command = new PutObjectCommand({\n      Bucket: this.config.s3.bucket,\n      Key: key,\n      Body: data,\n      ContentType: 'application/json',\n      Metadata: {\n        'trace-version': '1.0',\n        'compression': 'true',\n      },\n    });\n    \n    await this.s3Client.send(command);\n  }\n  \n  /**\n   * Record migration in local database\n   */\n  private recordMigration(\n    candidate: MigrationCandidate,\n    s3Key: string,\n    originalData: any,\n    compressedData: Buffer\n  ): void {\n    const stmt = this.localDb.prepare(`\n      INSERT INTO remote_migrations (\n        trace_id, migrated_at, storage_tier, s3_key,\n        compression_level, original_size, compressed_size\n      ) VALUES (?, ?, ?, ?, ?, ?, ?)\n    `);\n    \n    stmt.run(\n      candidate.traceId,\n      Date.now(),\n      candidate.tier,\n      s3Key,\n      candidate.compressionLevel,\n      JSON.stringify(originalData).length,\n      compressedData.length\n    );\n  }\n  \n  /**\n   * Remove local trace after migration\n   */\n  private removeLocalTrace(traceId: string): void {\n    this.localDb.prepare('DELETE FROM tool_calls WHERE trace_id = ?').run(traceId);\n    this.localDb.prepare('DELETE FROM traces WHERE id = ?').run(traceId);\n  }\n  \n  /**\n   * Get current local storage size\n   */\n  private getLocalStorageSize(): number {\n    const result = this.localDb.prepare(`\n      SELECT \n        SUM(LENGTH(compressed_data)) +\n        COALESCE((SELECT SUM(LENGTH(arguments) + LENGTH(result)) \n                  FROM tool_calls), 0) as total_size\n      FROM traces\n    `).get() as any;\n    \n    return result?.total_size || 0;\n  }\n  \n  /**\n   * Create batches from candidates\n   */\n  private createBatches<T>(items: T[], batchSize: number): T[][] {\n    const batches: T[][] = [];\n    for (let i = 0; i < items.length; i += batchSize) {\n      batches.push(items.slice(i, i + batchSize));\n    }\n    return batches;\n  }\n  \n  /**\n   * Retrieve trace from remote storage\n   */\n  async retrieveTrace(traceId: string): Promise<any> {\n    const migration = this.localDb.prepare(`\n      SELECT * FROM remote_migrations WHERE trace_id = ?\n    `).get(traceId) as any;\n    \n    if (!migration) {\n      throw new Error(`Trace ${traceId} not found in remote storage`);\n    }\n    \n    // Update retrieval count\n    this.localDb.prepare(`\n      UPDATE remote_migrations \n      SET retrieval_count = retrieval_count + 1, last_retrieved = ?\n      WHERE trace_id = ?\n    `).run(Date.now(), traceId);\n    \n    if (!this.s3Client) {\n      throw new Error('S3 client not configured');\n    }\n    \n    // Retrieve from S3\n    const command = new GetObjectCommand({\n      Bucket: this.config.s3.bucket,\n      Key: migration.s3_key,\n    });\n    \n    const response = await this.s3Client.send(command);\n    const data = await response.Body?.transformToString();\n    \n    if (!data) {\n      throw new Error('No data retrieved from S3');\n    }\n    \n    return JSON.parse(data);\n  }\n  \n  /**\n   * Get migration statistics\n   */\n  getMigrationStats(): any {\n    const stats = this.localDb.prepare(`\n      SELECT \n        storage_tier,\n        COUNT(*) as count,\n        SUM(original_size) as original_size,\n        SUM(compressed_size) as compressed_size,\n        AVG(retrieval_count) as avg_retrievals\n      FROM remote_migrations\n      GROUP BY storage_tier\n    `).all();\n    \n    const total = this.localDb.prepare(`\n      SELECT \n        COUNT(*) as total_migrated,\n        SUM(original_size) as total_original,\n        SUM(compressed_size) as total_compressed\n      FROM remote_migrations\n    `).get();\n    \n    return {\n      byTier: stats,\n      total,\n      compressionRatio: total \n        ? (1 - (total as any).total_compressed / (total as any).total_original).toFixed(2)\n        : 0,\n      localSize: this.getLocalStorageSize(),\n    };\n  }\n}"],
+  "mappings": "AAKA,SAAS,UAAU,kBAAkB,wBAA6C;AAClF,SAAS,eAAe;AACxB,SAAS,cAAc;AAIhB,IAAK,cAAL,kBAAKA,iBAAL;AACL,EAAAA,aAAA,SAAM;AACN,EAAAA,aAAA,cAAW;AACX,EAAAA,aAAA,cAAW;AACX,EAAAA,aAAA,aAAU;AAJA,SAAAA;AAAA,GAAA;AAuCL,MAAM,wBAA6C;AAAA,EACxD,UAAU;AAAA;AAAA,EACV,KAAK;AAAA,IACH,YAAY;AAAA,IACZ,WAAW,QAAQ,IAAI,kBAAkB;AAAA,EAC3C;AAAA,EACA,YAAY;AAAA,IACV,MAAM;AAAA;AAAA,IACN,MAAM;AAAA,IACN,MAAM;AAAA,IACN,UAAU;AAAA,EACZ;AAAA,EACA,WAAW;AAAA,IACT,WAAW;AAAA,IACX,aAAa;AAAA;AAAA,IACb,kBAAkB;AAAA;AAAA,IAClB,kBAAkB;AAAA;AAAA,IAClB,iBAAiB;AAAA;AAAA,IACjB,gBAAgB;AAAA,EAClB;AACF;AAeO,MAAM,qBAAqB;AAAA,EACxB;AAAA,EACA;AAAA,EACA;AAAA,EACA,sBAAsB;AAAA,EAE9B,YACE,SACA,QACA;AACA,SAAK,UAAU;AACf,SAAK,SAAS,EAAE,GAAG,uBAAuB,GAAG,OAAO;AAEpD,SAAK,wBAAwB;AAC7B,SAAK,iBAAiB;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKQ,0BAAgC;AACtC,YAAQ,KAAK,OAAO,UAAU;AAAA,MAC5B,KAAK;AACH,YAAI,KAAK,OAAO,KAAK;AACnB,eAAK,gBAAgB,IAAI,QAAQ;AAAA,YAC/B,WAAW,KAAK,OAAO,IAAI;AAAA,YAC3B,aAAa,KAAK,OAAO,IAAI;AAAA,UAC/B,CAAC;AAAA,QACH;AACA;AAAA,MAEF,KAAK;AAAA,MACL,KAAK;AACH,YAAI,KAAK,OAAO,IAAI,eAAe,KAAK,OAAO,IAAI,iBAAiB;AAClE,eAAK,gBAAgB,IAAI,SAAS;AAAA,YAChC,QAAQ,KAAK,OAAO,GAAG;AAAA,YACvB,aAAa;AAAA,cACX,aAAa,KAAK,OAAO,GAAG;AAAA,cAC5B,iBAAiB,KAAK,OAAO,GAAG;AAAA,YAClC;AAAA,YACA,UAAU,KAAK,OAAO,GAAG;AAAA;AAAA,UAC3B,CAAC;AAAA,QACH;AACA;AAAA,IACJ;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,mBAAyB;AAC/B,SAAK,QAAQ,KAAK;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAcjB;AAED,SAAK,QAAQ,KAAK;AAAA;AAAA;AAAA,KAGjB;AAAA,EACH;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,8BAA6D;AACjE,UAAM,MAAM,KAAK,IAAI;AAGrB,UAAM,SAAS,KAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAanC,EAAE,IAAI;AAEP,UAAM,aAAmC,CAAC;AAE1C,eAAW,SAAS,QAAQ;AAC1B,YAAM,YAAY,MAAM,MAAM,eAAe,MAAO,KAAK;AACzD,YAAM,YAAY,KAAK;AAAA,QACrB,MAAM;AAAA,QACN;AAAA,QACA,MAAM;AAAA,QACN,MAAM,QAAQ;AAAA,MAChB;AAEA,iBAAW,KAAK,SAAS;AAAA,IAC3B;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,cACN,SACA,UACA,OACA,MACoB;AACpB,QAAI,OAAO;AACX,QAAI,gBAAgB;AACpB,QAAI,mBAA0D;AAG9D,QAAI,WAAW,KAAK,OAAO,UAAU,iBAAiB;AAEpD,aAAO;AACP,sBAAgB;AAChB,yBAAmB;AAAA,IACrB,WAAW,WAAW,KAAK,OAAO,UAAU,kBAAkB;AAE5D,aAAO;AACP,sBAAgB;AAChB,yBAAmB;AAAA,IACrB,WAAW,WAAW,KAAK,OAAO,UAAU,kBAAkB;AAE5D,aAAO;AACP,sBAAgB;AAChB,yBAAmB;AAAA,IACrB,WAAW,WAAW,KAAK,OAAO,UAAU,aAAa;AAEvD,aAAO;AACP,UAAI,QAAQ,KAAK,OAAO,UAAU,gBAAgB;AAChD,wBAAgB;AAChB,2BAAmB;AAAA,MACrB;AAAA,IACF;AAGA,UAAM,iBAAiB,IAAI,OAAO,OAAO;AACzC,UAAM,mBAAmB,KAAK,oBAAoB;AAElD,QAAI,mBAAmB,iBAAiB,MAAM;AAE5C,sBAAgB;AAChB,UAAI,qBAAqB,QAAQ;AAC/B,2BAAmB;AAAA,MACrB;AAAA,IACF;AAEA,WAAO;AAAA,MACL;AAAA,MACA,KAAK;AAAA,MACL;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,MACA;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,cACJ,YACA,SAAkB,OAMjB;AACD,QAAI,KAAK,qBAAqB;AAC5B,aAAO;AAAA,QACL,UAAU;AAAA,QACV,QAAQ;AAAA,QACR,WAAW;AAAA,QACX,QAAQ,CAAC,+BAA+B;AAAA,MAC1C;AAAA,IACF;AAEA,SAAK,sBAAsB;AAC3B,UAAM,UAAU;AAAA,MACd,UAAU;AAAA,MACV,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,QAAQ,CAAC;AAAA,IACX;AAEA,QAAI;AAEF,YAAM,YAAY,WAAW,OAAO,OAAK,EAAE,aAAa;AACxD,YAAM,UAAU,KAAK,cAAc,WAAW,KAAK,OAAO,UAAU,SAAS;AAE7E,iBAAW,SAAS,SAAS;AAC3B,YAAI,QAAQ;AACV,iBAAO,KAAK,iCAAiC;AAAA,YAC3C,OAAO,MAAM;AAAA,YACb,WAAW,MAAM,OAAO,CAAC,KAAK,MAAM,MAAM,EAAE,MAAM,CAAC;AAAA,UACrD,CAAC;AACD,kBAAQ,YAAY,MAAM;AAC1B;AAAA,QACF;AAEA,cAAM,eAAe,MAAM,KAAK,aAAa,KAAK;AAClD,gBAAQ,YAAY,aAAa;AACjC,gBAAQ,UAAU,aAAa;AAC/B,gBAAQ,aAAa,aAAa;AAClC,gBAAQ,OAAO,KAAK,GAAG,aAAa,MAAM;AAG1C,cAAM,IAAI,QAAQ,aAAW,WAAW,SAAS,GAAG,CAAC;AAAA,MACvD;AAAA,IAEF,UAAE;AACA,WAAK,sBAAsB;AAAA,IAC7B;AAEA,WAAO,KAAK,uBAAuB,OAAO;AAC1C,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,aACZ,OAMC;AACD,UAAM,UAAU;AAAA,MACd,SAAS;AAAA,MACT,QAAQ;AAAA,MACR,WAAW;AAAA,MACX,QAAQ,CAAC;AAAA,IACX;AAEA,eAAW,aAAa,OAAO;AAC7B,UAAI;AAEF,cAAM,QAAQ,KAAK,aAAa,UAAU,OAAO;AACjD,YAAI,CAAC,OAAO;AACV,gBAAM,IAAI,MAAM,SAAS,UAAU,OAAO,YAAY;AAAA,QACxD;AAGA,cAAM,aAAa,KAAK,cAAc,OAAO,UAAU,gBAAgB;AAGvE,YAAI,KAAK,UAAU;AACjB,gBAAM,QAAQ,KAAK,cAAc,SAAS;AAC1C,gBAAM,KAAK,WAAW,OAAO,UAAU;AAGvC,eAAK,gBAAgB,WAAW,OAAO,OAAO,UAAU;AAAA,QAC1D,OAAO;AAEL,eAAK,gBAAgB,WAAW,aAAa,OAAO,UAAU;AAAA,QAChE;AAGA,YAAI,UAAU,SAAS,YAAY,QAC/B,UAAU,SAAS,yBAAqB;AAC1C,eAAK,iBAAiB,UAAU,OAAO;AAAA,QACzC;AAEA,gBAAQ;AACR,gBAAQ,aAAa,UAAU;AAAA,MAEjC,SAAS,OAAO;AACd,gBAAQ;AACR,gBAAQ,OAAO;AAAA,UACb,qBAAqB,UAAU,OAAO,KAAK,KAAK;AAAA,QAClD;AACA,eAAO,MAAM,8BAA8B;AAAA,UACzC,SAAS,UAAU;AAAA,UACnB;AAAA,QACF,CAAC;AAAA,MACH;AAAA,IACF;AAEA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKQ,aAAa,SAAsB;AACzC,UAAM,WAAW,KAAK,QAAQ;AAAA,MAC5B;AAAA,IACF,EAAE,IAAI,OAAO;AAEb,QAAI,CAAC,SAAU,QAAO;AAEtB,UAAM,YAAY,KAAK,QAAQ;AAAA,MAC7B;AAAA,IACF,EAAE,IAAI,OAAO;AAEb,WAAO;AAAA,MACL,OAAO;AAAA,MACP;AAAA,IACF;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,cACN,MACA,OACQ;AACR,QAAI,WAAW,KAAK,UAAU,IAAI;AAGlC,YAAQ,OAAO;AAAA,MACb,KAAK;AACH,eAAO,OAAO,KAAK,QAAQ;AAAA,MAE7B,KAAK;AAEH,eAAO,OAAO,KAAK,KAAK,UAAU,KAAK,MAAM,QAAQ,CAAC,CAAC;AAAA,MAEzD,KAAK;AAEH,cAAM,UAAU,KAAK;AAAA,UAAM;AAAA,UAAU,CAAC,KAAK,UACzC,UAAU,QAAQ,UAAU,SAAY,SAAY;AAAA,QACtD;AACA,eAAO,OAAO,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,MAE5C,KAAK;AAEH,cAAM,UAAU;AAAA,UACd,IAAI,KAAK,MAAM;AAAA,UACf,MAAM,KAAK,MAAM;AAAA,UACjB,OAAO,KAAK,MAAM;AAAA,UAClB,SAAS,KAAK,MAAM;AAAA,UACpB,YAAY;AAAA,YACV,OAAO,KAAK,MAAM;AAAA,YAClB,KAAK,KAAK,MAAM;AAAA,UAClB;AAAA,UACA,WAAW,KAAK,UAAU;AAAA,UAC1B,WAAW,CAAC,GAAG,IAAI,IAAI,KAAK,UAAU,IAAI,CAAC,MAAW,EAAE,IAAI,CAAC,CAAC;AAAA,QAChE;AACA,eAAO,OAAO,KAAK,KAAK,UAAU,OAAO,CAAC;AAAA,MAE5C;AACE,eAAO,OAAO,KAAK,QAAQ;AAAA,IAC/B;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,cAAc,WAAuC;AAC3D,UAAM,OAAO,IAAI,KAAK,KAAK,IAAI,IAAI,UAAU,MAAM,KAAK,KAAK,GAAI;AACjE,UAAM,OAAO,KAAK,YAAY;AAC9B,UAAM,QAAQ,OAAO,KAAK,SAAS,IAAI,CAAC,EAAE,SAAS,GAAG,GAAG;AACzD,UAAM,MAAM,OAAO,KAAK,QAAQ,CAAC,EAAE,SAAS,GAAG,GAAG;AAElD,WAAO,UAAU,IAAI,IAAI,KAAK,IAAI,GAAG,IAAI,UAAU,IAAI,IAAI,UAAU,OAAO;AAAA,EAC9E;AAAA;AAAA;AAAA;AAAA,EAKA,MAAc,WAAW,KAAa,MAA6B;AACjE,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAEA,UAAM,UAAU,IAAI,iBAAiB;AAAA,MACnC,QAAQ,KAAK,OAAO,GAAG;AAAA,MACvB,KAAK;AAAA,MACL,MAAM;AAAA,MACN,aAAa;AAAA,MACb,UAAU;AAAA,QACR,iBAAiB;AAAA,QACjB,eAAe;AAAA,MACjB;AAAA,IACF,CAAC;AAED,UAAM,KAAK,SAAS,KAAK,OAAO;AAAA,EAClC;AAAA;AAAA;AAAA;AAAA,EAKQ,gBACN,WACA,OACA,cACA,gBACM;AACN,UAAM,OAAO,KAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA,KAKjC;AAED,SAAK;AAAA,MACH,UAAU;AAAA,MACV,KAAK,IAAI;AAAA,MACT,UAAU;AAAA,MACV;AAAA,MACA,UAAU;AAAA,MACV,KAAK,UAAU,YAAY,EAAE;AAAA,MAC7B,eAAe;AAAA,IACjB;AAAA,EACF;AAAA;AAAA;AAAA;AAAA,EAKQ,iBAAiB,SAAuB;AAC9C,SAAK,QAAQ,QAAQ,2CAA2C,EAAE,IAAI,OAAO;AAC7E,SAAK,QAAQ,QAAQ,iCAAiC,EAAE,IAAI,OAAO;AAAA,EACrE;AAAA;AAAA;AAAA;AAAA,EAKQ,sBAA8B;AACpC,UAAM,SAAS,KAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAMnC,EAAE,IAAI;AAEP,WAAO,QAAQ,cAAc;AAAA,EAC/B;AAAA;AAAA;AAAA;AAAA,EAKQ,cAAiB,OAAY,WAA0B;AAC7D,UAAM,UAAiB,CAAC;AACxB,aAAS,IAAI,GAAG,IAAI,MAAM,QAAQ,KAAK,WAAW;AAChD,cAAQ,KAAK,MAAM,MAAM,GAAG,IAAI,SAAS,CAAC;AAAA,IAC5C;AACA,WAAO;AAAA,EACT;AAAA;AAAA;AAAA;AAAA,EAKA,MAAM,cAAc,SAA+B;AACjD,UAAM,YAAY,KAAK,QAAQ,QAAQ;AAAA;AAAA,KAEtC,EAAE,IAAI,OAAO;AAEd,QAAI,CAAC,WAAW;AACd,YAAM,IAAI,MAAM,SAAS,OAAO,8BAA8B;AAAA,IAChE;AAGA,SAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA,KAIpB,EAAE,IAAI,KAAK,IAAI,GAAG,OAAO;AAE1B,QAAI,CAAC,KAAK,UAAU;AAClB,YAAM,IAAI,MAAM,0BAA0B;AAAA,IAC5C;AAGA,UAAM,UAAU,IAAI,iBAAiB;AAAA,MACnC,QAAQ,KAAK,OAAO,GAAG;AAAA,MACvB,KAAK,UAAU;AAAA,IACjB,CAAC;AAED,UAAM,WAAW,MAAM,KAAK,SAAS,KAAK,OAAO;AACjD,UAAM,OAAO,MAAM,SAAS,MAAM,kBAAkB;AAEpD,QAAI,CAAC,MAAM;AACT,YAAM,IAAI,MAAM,2BAA2B;AAAA,IAC7C;AAEA,WAAO,KAAK,MAAM,IAAI;AAAA,EACxB;AAAA;AAAA;AAAA;AAAA,EAKA,oBAAyB;AACvB,UAAM,QAAQ,KAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KASlC,EAAE,IAAI;AAEP,UAAM,QAAQ,KAAK,QAAQ,QAAQ;AAAA;AAAA;AAAA;AAAA;AAAA;AAAA,KAMlC,EAAE,IAAI;AAEP,WAAO;AAAA,MACL,QAAQ;AAAA,MACR;AAAA,MACA,kBAAkB,SACb,IAAK,MAAc,mBAAoB,MAAc,gBAAgB,QAAQ,CAAC,IAC/E;AAAA,MACJ,WAAW,KAAK,oBAAoB;AAAA,IACtC;AAAA,EACF;AACF;",
+  "names": ["StorageTier"]
+}

package/dist/core/trace/trace-detector.js

@@ -2,7 +2,8 @@ import { v4 as uuidv4 } from "uuid";
 import {
   TraceType,
   DEFAULT_TRACE_CONFIG,
-  TRACE_PATTERNS
+  TRACE_PATTERNS,
+  CompressionStrategy
 } from "./types.js";
 import { ConfigManager } from "../config/config-manager.js";
 import { TraceStore } from "./trace-store.js";
@@ -280,9 +281,41 @@ class TraceDetector {
     }
   }
   /**
-   * Compress a trace for long-term storage
+   * Compress a trace for long-term storage using strategy
    */
-  compressTrace(trace) {
+  compressTrace(trace, strategy = CompressionStrategy.PATTERN_BASED) {
+    switch (strategy) {
+      case CompressionStrategy.SUMMARY_ONLY:
+        return this.compressSummaryOnly(trace);
+      case CompressionStrategy.PATTERN_BASED:
+        return this.compressPatternBased(trace);
+      case CompressionStrategy.SELECTIVE:
+        return this.compressSelective(trace);
+      case CompressionStrategy.FULL_COMPRESSION:
+        return this.compressMaximal(trace);
+      default:
+        return this.compressPatternBased(trace);
+    }
+  }
+  /**
+   * Summary-only compression - minimal data retention
+   */
+  compressSummaryOnly(trace) {
+    return {
+      pattern: "",
+      // No pattern stored
+      summary: trace.summary.substring(0, 100),
+      // Limit summary
+      score: trace.score,
+      toolCount: trace.tools.length,
+      duration: trace.metadata.endTime - trace.metadata.startTime,
+      timestamp: trace.metadata.startTime
+    };
+  }
+  /**
+   * Pattern-based compression - keep tool sequence
+   */
+  compressPatternBased(trace) {
     const pattern = trace.tools.map((t) => t.tool).join("\u2192");
     const duration = trace.metadata.endTime - trace.metadata.startTime;
     return {
@@ -294,6 +327,84 @@ class TraceDetector {
       timestamp: trace.metadata.startTime
     };
   }
+  /**
+   * Selective compression - keep high-score tools only
+   */
+  compressSelective(trace, threshold = 0.5) {
+    const significantTools = trace.tools.filter((tool) => {
+      const score = this.configManager.calculateScore(tool.tool, {
+        filesAffected: tool.filesAffected?.length || 0,
+        isPermanent: this.isPermanentChange(tool),
+        referenceCount: 0
+      });
+      return score >= threshold;
+    });
+    const pattern = significantTools.length > 0 ? significantTools.map((t) => t.tool).join("\u2192") : trace.tools.map((t) => t.tool).join("\u2192");
+    return {
+      pattern,
+      summary: `${trace.summary} [${significantTools.length}/${trace.tools.length} significant]`,
+      score: trace.score,
+      toolCount: significantTools.length,
+      duration: trace.metadata.endTime - trace.metadata.startTime,
+      timestamp: trace.metadata.startTime
+    };
+  }
+  /**
+   * Maximal compression - absolute minimum data
+   */
+  compressMaximal(trace) {
+    const typeAbbrev = this.getTraceTypeAbbreviation(trace.type);
+    const pattern = `${typeAbbrev}:${trace.tools.length}`;
+    return {
+      pattern,
+      summary: trace.type,
+      // Just the type
+      score: Math.round(trace.score * 10) / 10,
+      // Round to 1 decimal
+      toolCount: trace.tools.length,
+      duration: Math.round((trace.metadata.endTime - trace.metadata.startTime) / 1e3) * 1e3,
+      // Round to seconds
+      timestamp: trace.metadata.startTime
+    };
+  }
+  /**
+   * Get abbreviated trace type
+   */
+  getTraceTypeAbbreviation(type) {
+    const abbreviations = {
+      [TraceType.SEARCH_DRIVEN]: "SD",
+      [TraceType.ERROR_RECOVERY]: "ER",
+      [TraceType.FEATURE_IMPLEMENTATION]: "FI",
+      [TraceType.REFACTORING]: "RF",
+      [TraceType.TESTING]: "TS",
+      [TraceType.EXPLORATION]: "EX",
+      [TraceType.DEBUGGING]: "DB",
+      [TraceType.DOCUMENTATION]: "DC",
+      [TraceType.BUILD_DEPLOY]: "BD",
+      [TraceType.UNKNOWN]: "UN"
+    };
+    return abbreviations[type] || "UN";
+  }
+  /**
+   * Choose compression strategy based on trace age and importance
+   */
+  selectCompressionStrategy(trace) {
+    const ageHours = (Date.now() - trace.metadata.startTime) / (1e3 * 60 * 60);
+    const score = trace.score;
+    if (ageHours < 24 && score > 0.7) {
+      return CompressionStrategy.PATTERN_BASED;
+    }
+    if (ageHours < 24) {
+      return CompressionStrategy.SELECTIVE;
+    }
+    if (ageHours < 168 && score > 0.5) {
+      return CompressionStrategy.SELECTIVE;
+    }
+    if (ageHours < 720) {
+      return CompressionStrategy.SUMMARY_ONLY;
+    }
+    return CompressionStrategy.FULL_COMPRESSION;
+  }
   /**
    * Get directory from file path
    */
@@ -329,7 +440,7 @@ class TraceDetector {
     return this.traces.filter((t) => t.score >= threshold);
   }
   /**
-   * Compress old traces
+   * Compress old traces with intelligent strategy selection
    */
   compressOldTraces(ageHours = 24) {
     let compressed = 0;
@@ -337,8 +448,28 @@ class TraceDetector {
     for (const trace of this.traces) {
       const age = (now - trace.metadata.startTime) / (1e3 * 60 * 60);
       if (age > ageHours && !trace.compressed) {
-        trace.compressed = this.compressTrace(trace);
+        const strategy = this.selectCompressionStrategy(trace);
+        trace.compressed = this.compressTrace(trace, strategy);
+        if (strategy === CompressionStrategy.FULL_COMPRESSION || strategy === CompressionStrategy.SUMMARY_ONLY) {
+          trace.tools = [];
+        } else if (strategy === CompressionStrategy.SELECTIVE) {
+          trace.tools = trace.tools.filter((tool) => {
+            const score = this.configManager.calculateScore(tool.tool, {
+              filesAffected: tool.filesAffected?.length || 0,
+              isPermanent: this.isPermanentChange(tool),
+              referenceCount: 0
+            });
+            return score >= 0.5;
+          });
+        }
         compressed++;
+        if (this.traceStore) {
+          try {
+            this.traceStore.updateCompression(trace.id, trace.compressed, strategy);
+          } catch (error) {
+            console.error("Failed to update trace compression in store:", error);
+          }
+        }
       }
     }
     return compressed;