@stackmemoryai/stackmemory 0.5.57 → 0.5.59

package/dist/src/core/storage/two-tier-storage.js

@@ -0,0 +1,766 @@
+import { fileURLToPath as __fileURLToPath } from 'url';
+import { dirname as __pathDirname } from 'path';
+const __filename = __fileURLToPath(import.meta.url);
+const __dirname = __pathDirname(__filename);
+import Database from "better-sqlite3";
+import {
+  S3Client,
+  PutObjectCommand,
+  GetObjectCommand
+} from "@aws-sdk/client-s3";
+import { createClient as createRedisClient } from "redis";
+import { Pool } from "pg";
+import * as zlib from "zlib";
+import { promisify } from "util";
+import { v4 as uuidv4 } from "uuid";
+import { Logger } from "../monitoring/logger.js";
+const lz4 = {
+  encode: (data) => data,
+  // Placeholder
+  decode: (data) => data
+  // Placeholder
+};
+const gzip = promisify(zlib.gzip);
+const gunzip = promisify(zlib.gunzip);
+var StorageTier = /* @__PURE__ */ ((StorageTier2) => {
+  StorageTier2["YOUNG"] = "young";
+  StorageTier2["MATURE"] = "mature";
+  StorageTier2["OLD"] = "old";
+  StorageTier2["REMOTE"] = "remote";
+  return StorageTier2;
+})(StorageTier || {});
+class TwoTierStorageSystem {
+  db;
+  redisClient;
+  timeseriesPool;
+  s3Client;
+  logger;
+  config;
+  migrationTimer;
+  offlineQueue = [];
+  stats;
+  constructor(config) {
+    this.config = config;
+    this.logger = new Logger("TwoTierStorage");
+    this.db = new Database(config.local.dbPath);
+    this.initializeLocalStorage();
+    this.s3Client = new S3Client({
+      region: config.remote.s3.region,
+      credentials: config.remote.s3.accessKeyId && config.remote.s3.secretAccessKey ? {
+        accessKeyId: config.remote.s3.accessKeyId,
+        secretAccessKey: config.remote.s3.secretAccessKey
+      } : void 0
+    });
+    this.stats = {
+      localUsageMB: 0,
+      remoteUsageMB: 0,
+      tierDistribution: {
+        ["young" /* YOUNG */]: 0,
+        ["mature" /* MATURE */]: 0,
+        ["old" /* OLD */]: 0,
+        ["remote" /* REMOTE */]: 0
+      },
+      compressionRatio: 1,
+      migrationsPending: 0,
+      lastMigration: null
+    };
+  }
+  initializeLocalStorage() {
+    this.db.pragma("journal_mode = WAL");
+    this.db.pragma("synchronous = NORMAL");
+    this.db.exec(`
+      CREATE TABLE IF NOT EXISTS storage_items (
+        id TEXT PRIMARY KEY,
+        frame_id TEXT NOT NULL,
+        tier TEXT NOT NULL,
+        data BLOB NOT NULL,
+        metadata TEXT,
+        size_bytes INTEGER,
+        importance_score REAL DEFAULT 0.5,
+        access_count INTEGER DEFAULT 0,
+        last_accessed DATETIME DEFAULT CURRENT_TIMESTAMP,
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+        compressed BOOLEAN DEFAULT FALSE,
+        compression_type TEXT
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_tier_age ON storage_items (tier, created_at);
+      CREATE INDEX IF NOT EXISTS idx_frame ON storage_items (frame_id);
+      CREATE INDEX IF NOT EXISTS idx_importance ON storage_items (importance_score DESC);
+      
+      CREATE TABLE IF NOT EXISTS migration_queue (
+        id TEXT PRIMARY KEY,
+        item_id TEXT NOT NULL,
+        source_tier TEXT NOT NULL,
+        target_tier TEXT NOT NULL,
+        priority INTEGER DEFAULT 5,
+        attempts INTEGER DEFAULT 0,
+        status TEXT DEFAULT 'pending',
+        created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+      
+      CREATE INDEX IF NOT EXISTS idx_status_priority ON migration_queue (status, priority DESC);
+      
+      CREATE TABLE IF NOT EXISTS storage_metrics (
+        id INTEGER PRIMARY KEY AUTOINCREMENT,
+        tier TEXT NOT NULL,
+        item_count INTEGER,
+        total_size_mb REAL,
+        avg_compression_ratio REAL,
+        measured_at DATETIME DEFAULT CURRENT_TIMESTAMP
+      );
+    `);
+  }
+  async initialize() {
+    try {
+      if (this.config.remote.redis?.url) {
+        this.redisClient = createRedisClient({
+          url: this.config.remote.redis.url
+        });
+        await this.redisClient.connect();
+        this.logger.info("Redis connected for hot cache");
+      }
+      if (this.config.remote.timeseries?.connectionString) {
+        this.timeseriesPool = new Pool({
+          connectionString: this.config.remote.timeseries.connectionString,
+          max: 5
+        });
+        await this.initializeTimeseriesSchema();
+        this.logger.info("TimeSeries DB connected");
+      }
+      this.startMigrationWorker();
+      await this.loadOfflineQueue();
+      await this.updateStats();
+      this.logger.info("Two-tier storage system initialized");
+    } catch (error) {
+      this.logger.error("Failed to initialize storage", { error });
+      throw error;
+    }
+  }
+  async initializeTimeseriesSchema() {
+    if (!this.timeseriesPool) return;
+    const client = await this.timeseriesPool.connect();
+    try {
+      await client.query(`
+        CREATE TABLE IF NOT EXISTS frame_timeseries (
+          time TIMESTAMPTZ NOT NULL,
+          frame_id TEXT NOT NULL,
+          project_id TEXT NOT NULL,
+          data JSONB,
+          metrics JSONB,
+          importance_score REAL
+        );
+        
+        SELECT create_hypertable('frame_timeseries', 'time', 
+          chunk_time_interval => INTERVAL '1 month',
+          if_not_exists => TRUE
+        );
+        
+        CREATE INDEX IF NOT EXISTS idx_frame_time 
+          ON frame_timeseries (frame_id, time DESC);
+      `);
+    } finally {
+      client.release();
+    }
+  }
+  /**
+   * Store a frame with automatic tier selection
+   */
+  async storeFrame(frame, events, anchors) {
+    const storageId = uuidv4();
+    const data = { frame, events, anchors };
+    const tier = this.selectTier(frame);
+    const importanceScore = this.calculateImportance(frame, events, anchors);
+    const compressed = await this.compressData(data, tier);
+    const stmt = this.db.prepare(`
+      INSERT INTO storage_items (
+        id, frame_id, tier, data, metadata, size_bytes, 
+        importance_score, compressed, compression_type
+      ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+    `);
+    stmt.run(
+      storageId,
+      frame.frame_id,
+      tier,
+      compressed.data,
+      JSON.stringify({
+        originalSize: compressed.originalSize,
+        compressedSize: compressed.compressedSize
+      }),
+      compressed.compressedSize,
+      importanceScore,
+      compressed.compressed ? 1 : 0,
+      compressed.compressionType
+    );
+    if (tier === "young" /* YOUNG */ && this.redisClient) {
+      await this.redisClient.setex(
+        `frame:${frame.frame_id}`,
+        this.config.remote.redis?.ttlSeconds || 300,
+        JSON.stringify(data)
+      );
+    }
+    if (importanceScore > 0.7) {
+      await this.queueRemoteUpload(storageId, data, "high");
+    }
+    return storageId;
+  }
+  /**
+   * Retrieve a frame from any tier
+   */
+  async retrieveFrame(frameId) {
+    if (this.redisClient) {
+      try {
+        const cached = await this.redisClient.get(`frame:${frameId}`);
+        if (cached) {
+          this.updateAccessCount(frameId);
+          return JSON.parse(cached);
+        }
+      } catch (error) {
+        this.logger.warn("Redis retrieval failed", { frameId, error });
+      }
+    }
+    const local = this.db.prepare(
+      `
+      SELECT data, compressed, compression_type 
+      FROM storage_items 
+      WHERE frame_id = ?
+      ORDER BY created_at DESC
+      LIMIT 1
+    `
+    ).get(frameId);
+    if (local) {
+      this.updateAccessCount(frameId);
+      const data = local.compressed ? await this.decompressData(local.data, local.compression_type) : JSON.parse(local.data);
+      if (this.redisClient) {
+        await this.redisClient.setex(
+          `frame:${frameId}`,
+          300,
+          // 5 minute cache
+          JSON.stringify(data)
+        );
+      }
+      return data;
+    }
+    return this.retrieveFromRemote(frameId);
+  }
+  /**
+   * Select appropriate tier based on frame age and characteristics
+   */
+  selectTier(frame) {
+    if (!frame.created_at || isNaN(frame.created_at)) {
+      this.logger.warn("Invalid frame timestamp, defaulting to YOUNG tier");
+      return "young" /* YOUNG */;
+    }
+    const ageHours = (Date.now() - frame.created_at) / (1e3 * 60 * 60);
+    if (ageHours < 24) return "young" /* YOUNG */;
+    if (ageHours < 168) return "mature" /* MATURE */;
+    if (ageHours < 720) return "old" /* OLD */;
+    return "remote" /* REMOTE */;
+  }
+  /**
+   * Calculate importance score for migration decisions
+   */
+  calculateImportance(frame, events, anchors) {
+    let score = 0.5;
+    const decisions = anchors.filter((a) => a.type === "DECISION");
+    score += decisions.length * 0.1;
+    score += Math.min(events.length * 0.01, 0.2);
+    const ageHours = (Date.now() - frame.created_at) / (1e3 * 60 * 60);
+    if (ageHours < 24) score += 0.2;
+    else if (ageHours < 168) score += 0.1;
+    const errors = events.filter((e) => e.event_type === "error");
+    if (errors.length > 0) score += 0.2;
+    return Math.min(score, 1);
+  }
+  /**
+   * Compress data based on tier configuration
+   */
+  async compressData(data, tier) {
+    const json = JSON.stringify(data);
+    const originalSize = Buffer.byteLength(json);
+    const tierConfig = this.config.local.tiers.find((t) => t.name === tier);
+    if (!tierConfig || tierConfig.compressionType === "none") {
+      return {
+        data: Buffer.from(json),
+        originalSize,
+        compressedSize: originalSize,
+        compressed: false,
+        compressionType: "none"
+      };
+    }
+    let compressed;
+    switch (tierConfig.compressionType) {
+      case "lz4":
+        compressed = lz4.encode(Buffer.from(json));
+        break;
+      case "zstd":
+        compressed = await gzip(json, { level: 9 });
+        break;
+      case "gzip":
+        compressed = await gzip(json);
+        break;
+      default:
+        compressed = Buffer.from(json);
+    }
+    return {
+      data: compressed,
+      originalSize,
+      compressedSize: compressed.length,
+      compressed: true,
+      compressionType: tierConfig.compressionType
+    };
+  }
+  /**
+   * Decompress data
+   */
+  async decompressData(data, compressionType) {
+    let decompressed;
+    switch (compressionType) {
+      case "lz4":
+        decompressed = lz4.decode(data);
+        break;
+      case "zstd":
+      case "gzip":
+        decompressed = await gunzip(data);
+        break;
+      default:
+        decompressed = data;
+    }
+    return JSON.parse(decompressed.toString());
+  }
+  /**
+   * Start background migration worker
+   */
+  startMigrationWorker() {
+    this.migrationTimer = setInterval(async () => {
+      try {
+        await this.processMigrations();
+        await this.checkMigrationTriggers();
+        await this.processOfflineQueue();
+      } catch (error) {
+        this.logger.error("Migration worker error", { error });
+      }
+    }, this.config.migration.intervalMs);
+    this.logger.info("Migration worker started");
+  }
+  /**
+   * Process pending migrations
+   */
+  async processMigrations() {
+    const pending = this.db.prepare(
+      `
+      SELECT * FROM migration_queue 
+      WHERE status = 'pending' 
+      ORDER BY priority DESC, created_at ASC 
+      LIMIT ?
+    `
+    ).all(this.config.migration.batchSize);
+    for (const migration of pending) {
+      try {
+        await this.executeMigration(migration);
+        this.db.prepare(
+          `
+          UPDATE migration_queue 
+          SET status = 'completed' 
+          WHERE id = ?
+        `
+        ).run(migration.id);
+      } catch (error) {
+        this.logger.error("Migration failed", { migration, error });
+        this.db.prepare(
+          `
+          UPDATE migration_queue 
+          SET attempts = attempts + 1,
+              status = CASE WHEN attempts >= 3 THEN 'failed' ELSE 'pending' END
+          WHERE id = ?
+        `
+        ).run(migration.id);
+      }
+    }
+    this.stats.lastMigration = /* @__PURE__ */ new Date();
+  }
+  /**
+   * Check and trigger migrations based on rules
+   */
+  async checkMigrationTriggers() {
+    for (const trigger of this.config.migration.triggers) {
+      if (trigger.type === "age") {
+        const items = this.db.prepare(
+          `
+          SELECT id, frame_id, tier 
+          FROM storage_items 
+          WHERE julianday('now') - julianday(created_at) > ?
+          AND tier != ?
+          LIMIT 100
+        `
+        ).all(trigger.threshold / 24, "remote" /* REMOTE */);
+        for (const item of items) {
+          this.queueMigration(item.id, item.tier, "remote" /* REMOTE */);
+        }
+      }
+      if (trigger.type === "size") {
+        const stats = this.db.prepare(
+          `
+          SELECT SUM(size_bytes) as total_size 
+          FROM storage_items 
+          WHERE tier IN ('young', 'mature')
+        `
+        ).get();
+        if (stats.total_size > trigger.threshold * 1024 * 1024) {
+          const items = this.db.prepare(
+            `
+            SELECT id, tier FROM storage_items 
+            WHERE tier IN ('young', 'mature')
+            ORDER BY created_at ASC 
+            LIMIT 50
+          `
+          ).all();
+          for (const item of items) {
+            const targetTier = item.tier === "young" /* YOUNG */ ? "mature" /* MATURE */ : "old" /* OLD */;
+            this.queueMigration(item.id, item.tier, targetTier);
+          }
+        }
+      }
+    }
+  }
+  /**
+   * Queue a migration
+   */
+  queueMigration(itemId, sourceTier, targetTier, priority = 5) {
+    const id = uuidv4();
+    this.db.prepare(
+      `
+      INSERT INTO migration_queue (id, item_id, source_tier, target_tier, priority)
+      VALUES (?, ?, ?, ?, ?)
+    `
+    ).run(id, itemId, sourceTier, targetTier, priority);
+    this.stats.migrationsPending++;
+  }
+  /**
+   * Execute a single migration
+   */
+  async executeMigration(migration) {
+    const item = this.db.prepare(
+      `
+      SELECT * FROM storage_items WHERE id = ?
+    `
+    ).get(migration.item_id);
+    if (!item) {
+      throw new Error(`Item not found: ${migration.item_id}`);
+    }
+    const data = item.compressed ? await this.decompressData(item.data, item.compression_type) : JSON.parse(item.data);
+    if (migration.target_tier === "remote" /* REMOTE */) {
+      await this.uploadToS3(item.frame_id, data);
+      this.db.prepare(
+        `
+        DELETE FROM storage_items WHERE id = ?
+      `
+      ).run(migration.item_id);
+    } else {
+      const compressed = await this.compressData(data, migration.target_tier);
+      this.db.prepare(
+        `
+        UPDATE storage_items 
+        SET tier = ?, data = ?, size_bytes = ?, 
+            compressed = ?, compression_type = ?
+        WHERE id = ?
+      `
+      ).run(
+        migration.target_tier,
+        compressed.data,
+        compressed.compressedSize,
+        compressed.compressed ? 1 : 0,
+        compressed.compressionType,
+        migration.item_id
+      );
+    }
+    this.logger.info("Migration completed", {
+      itemId: migration.item_id,
+      from: migration.source_tier,
+      to: migration.target_tier
+    });
+  }
+  /**
+   * Upload data to S3
+   */
+  async uploadToS3(frameId, data) {
+    const date = /* @__PURE__ */ new Date();
+    const partition = `${date.getFullYear()}/${String(date.getMonth() + 1).padStart(2, "0")}`;
+    const key = `frames/${partition}/${frameId}.json.gz`;
+    const compressed = await gzip(JSON.stringify(data));
+    const command = new PutObjectCommand({
+      Bucket: this.config.remote.s3.bucket,
+      Key: key,
+      Body: compressed,
+      ContentType: "application/json",
+      ContentEncoding: "gzip",
+      Metadata: {
+        frameId,
+        uploadedAt: date.toISOString()
+      }
+    });
+    await this.s3Client.send(command);
+  }
+  /**
+   * Retrieve from remote storage
+   */
+  async retrieveFromRemote(frameId) {
+    if (this.timeseriesPool) {
+      const client = await this.timeseriesPool.connect();
+      try {
+        const result = await client.query(
+          `
+          SELECT data FROM frame_timeseries 
+          WHERE frame_id = $1 
+          ORDER BY time DESC 
+          LIMIT 1
+        `,
+          [frameId]
+        );
+        if (result.rows.length > 0) {
+          return result.rows[0].data;
+        }
+      } finally {
+        client.release();
+      }
+    }
+    const date = /* @__PURE__ */ new Date();
+    for (let i = 0; i < 3; i++) {
+      const checkDate = new Date(date);
+      checkDate.setMonth(checkDate.getMonth() - i);
+      const partition = `${checkDate.getFullYear()}/${String(checkDate.getMonth() + 1).padStart(2, "0")}`;
+      const key = `frames/${partition}/${frameId}.json.gz`;
+      try {
+        const command = new GetObjectCommand({
+          Bucket: this.config.remote.s3.bucket,
+          Key: key
+        });
+        const response = await this.s3Client.send(command);
+        if (!response.Body) continue;
+        const body = await response.Body.transformToByteArray();
+        const decompressed = await gunzip(Buffer.from(body));
+        return JSON.parse(decompressed.toString());
+      } catch {
+      }
+    }
+    return null;
+  }
+  /**
+   * Queue for offline upload
+   */
+  async queueRemoteUpload(id, data, priority) {
+    this.offlineQueue.push({
+      id,
+      data,
+      priority,
+      timestamp: Date.now()
+    });
+    if (this.config.migration.offlineQueuePath) {
+      await this.saveOfflineQueue();
+    }
+  }
+  /**
+   * Process offline upload queue
+   */
+  async processOfflineQueue() {
+    if (this.offlineQueue.length === 0) return;
+    const isOnline = await this.checkConnectivity();
+    if (!isOnline) return;
+    this.offlineQueue.sort((a, b) => {
+      if (a.priority !== b.priority) {
+        return a.priority === "high" ? -1 : 1;
+      }
+      return a.timestamp - b.timestamp;
+    });
+    const batch = this.offlineQueue.splice(0, 10);
+    for (const item of batch) {
+      try {
+        await this.uploadToS3(item.id, item.data);
+      } catch {
+        this.offlineQueue.push(item);
+      }
+    }
+    await this.saveOfflineQueue();
+  }
+  /**
+   * Check connectivity to remote services
+   */
+  async checkConnectivity() {
+    try {
+      const response = await fetch("https://s3.amazonaws.com");
+      return response.ok;
+    } catch {
+      return false;
+    }
+  }
+  /**
+   * Update access count for cache promotion
+   */
+  updateAccessCount(frameId) {
+    this.db.prepare(
+      `
+      UPDATE storage_items 
+      SET access_count = access_count + 1,
+          last_accessed = CURRENT_TIMESTAMP
+      WHERE frame_id = ?
+    `
+    ).run(frameId);
+  }
+  /**
+   * Save offline queue to disk
+   */
+  async saveOfflineQueue() {
+    if (!this.config.migration.offlineQueuePath) return;
+    const fs = await import("fs/promises");
+    await fs.writeFile(
+      this.config.migration.offlineQueuePath,
+      JSON.stringify(this.offlineQueue),
+      "utf-8"
+    );
+  }
+  /**
+   * Load offline queue from disk
+   */
+  async loadOfflineQueue() {
+    if (!this.config.migration.offlineQueuePath) return;
+    const fs = await import("fs/promises");
+    try {
+      const data = await fs.readFile(
+        this.config.migration.offlineQueuePath,
+        "utf-8"
+      );
+      this.offlineQueue = JSON.parse(data);
+      this.logger.info(
+        `Loaded ${this.offlineQueue.length} items from offline queue`
+      );
+    } catch {
+    }
+  }
+  /**
+   * Update storage statistics
+   */
+  async updateStats() {
+    const localStats = this.db.prepare(
+      `
+      SELECT 
+        tier,
+        COUNT(*) as count,
+        SUM(size_bytes) / 1048576.0 as size_mb
+      FROM storage_items
+      GROUP BY tier
+    `
+    ).all();
+    this.stats.localUsageMB = 0;
+    for (const stat of localStats) {
+      this.stats.tierDistribution[stat.tier] = stat.count;
+      this.stats.localUsageMB += stat.size_mb;
+    }
+    const compressionStats = this.db.prepare(
+      `
+      SELECT 
+        AVG(CAST(json_extract(metadata, '$.originalSize') AS REAL) / 
+            CAST(json_extract(metadata, '$.compressedSize') AS REAL)) as ratio
+      FROM storage_items
+      WHERE compressed = 1
+    `
+    ).get();
+    this.stats.compressionRatio = compressionStats?.ratio || 1;
+    const pending = this.db.prepare(
+      `
+      SELECT COUNT(*) as count 
+      FROM migration_queue 
+      WHERE status = 'pending'
+    `
+    ).get();
+    this.stats.migrationsPending = pending.count;
+    this.db.prepare(
+      `
+      INSERT INTO storage_metrics (tier, item_count, total_size_mb, avg_compression_ratio)
+      VALUES ('all', ?, ?, ?)
+    `
+    ).run(
+      Object.values(this.stats.tierDistribution).reduce((a, b) => a + b, 0),
+      this.stats.localUsageMB,
+      this.stats.compressionRatio
+    );
+  }
+  /**
+   * Get current storage statistics
+   */
+  async getStats() {
+    await this.updateStats();
+    return { ...this.stats };
+  }
+  /**
+   * Cleanup and shutdown
+   */
+  async shutdown() {
+    if (this.migrationTimer) {
+      clearInterval(this.migrationTimer);
+    }
+    await this.saveOfflineQueue();
+    if (this.redisClient) {
+      await this.redisClient.quit();
+    }
+    if (this.timeseriesPool) {
+      await this.timeseriesPool.end();
+    }
+    this.db.close();
+    this.logger.info("Two-tier storage system shut down");
+  }
+}
+const defaultTwoTierConfig = {
+  local: {
+    dbPath: "~/.stackmemory/two-tier.db",
+    maxSizeGB: 2,
+    tiers: [
+      {
+        name: "young" /* YOUNG */,
+        maxAgeHours: 24,
+        compressionType: "none",
+        retentionPolicy: "complete",
+        maxSizeMB: 500
+      },
+      {
+        name: "mature" /* MATURE */,
+        maxAgeHours: 168,
+        compressionType: "lz4",
+        retentionPolicy: "selective",
+        maxSizeMB: 1e3
+      },
+      {
+        name: "old" /* OLD */,
+        maxAgeHours: 720,
+        compressionType: "zstd",
+        retentionPolicy: "critical",
+        maxSizeMB: 500
+      }
+    ]
+  },
+  remote: {
+    s3: {
+      bucket: process.env.S3_BUCKET || "stackmemory-storage",
+      region: process.env.AWS_REGION || "us-east-1"
+    }
+  },
+  migration: {
+    triggers: [
+      { type: "age", threshold: 720, action: "migrate" },
+      // 30 days
+      { type: "size", threshold: 1500, action: "migrate" },
+      // 1.5GB
+      { type: "importance", threshold: 0.3, action: "delete" }
+      // Low importance
+    ],
+    batchSize: 50,
+    intervalMs: 6e4,
+    // 1 minute
+    offlineQueuePath: "~/.stackmemory/offline-queue.json"
+  }
+};
+export {
+  StorageTier,
+  TwoTierStorageSystem,
+  defaultTwoTierConfig
+};
+//# sourceMappingURL=two-tier-storage.js.map