@deimoscloud/coreai 0.1.0

package/src/cache/provider.ts

@@ -0,0 +1,745 @@
+/**
+ * File-based Cache Provider
+ *
+ * Implements CacheProvider using the local filesystem.
+ * Stores content in files with separate metadata JSON files.
+ */
+
+import { promises as fs } from 'fs';
+import { join, dirname } from 'path';
+import { createHash } from 'crypto';
+import type {
+  CacheEntry,
+  CacheMetadata,
+  CacheOptions,
+  CacheListOptions,
+  CacheStats,
+  CacheStatus,
+  CacheSource,
+} from './types.js';
+import { CacheError, CACHE_PATHS, DEFAULT_CACHE_CONFIG } from './types.js';
+import type { CacheProvider } from './interfaces.js';
+
+/**
+ * No-op function for ignoring errors
+ */
+function noop(): void {
+  // Intentionally empty
+}
+
+/**
+ * Options for creating a file cache provider
+ */
+export interface FileCacheProviderOptions {
+  /**
+   * Base path for cache storage (project root)
+   */
+  basePath: string;
+
+  /**
+   * Default TTL in seconds
+   */
+  ttl?: number;
+
+  /**
+   * Maximum cache size in bytes
+   */
+  maxSize?: number;
+
+  /**
+   * Maximum number of entries
+   */
+  maxEntries?: number;
+}
+
+/**
+ * Cache index structure for fast lookups
+ */
+interface CacheIndex {
+  version: number;
+  entries: Record<string, CacheIndexEntry>;
+  stats: {
+    totalSize: number;
+    entryCount: number;
+    lastCleanup: string | null;
+  };
+}
+
+/**
+ * Index entry for quick access
+ */
+interface CacheIndexEntry {
+  key: string;
+  source: CacheSource;
+  cachedAt: string;
+  expiresAt: string;
+  size: number;
+  contentFile: string;
+  metadataFile: string;
+}
+
+const INDEX_VERSION = 1;
+
+/**
+ * File-based cache provider implementation
+ */
+export class FileCacheProvider implements CacheProvider {
+  private basePath: string;
+  private cachePath: string;
+  private contentPath: string;
+  private metadataPath: string;
+  private indexPath: string;
+  private ttl: number;
+  private maxSize: number;
+  private maxEntries: number;
+  private initialized = false;
+  private index: CacheIndex | null = null;
+
+  constructor(options: FileCacheProviderOptions) {
+    this.basePath = options.basePath;
+    this.cachePath = join(this.basePath, CACHE_PATHS.ROOT);
+    this.contentPath = join(this.basePath, CACHE_PATHS.CONTENT);
+    this.metadataPath = join(this.basePath, CACHE_PATHS.METADATA);
+    this.indexPath = join(this.basePath, CACHE_PATHS.INDEX);
+    this.ttl = options.ttl ?? DEFAULT_CACHE_CONFIG.ttl;
+    this.maxSize = options.maxSize ?? DEFAULT_CACHE_CONFIG.maxSize;
+    this.maxEntries = options.maxEntries ?? DEFAULT_CACHE_CONFIG.maxEntries;
+  }
+
+  /**
+   * Initialize the cache (create directories, load index)
+   */
+  async initialize(): Promise<void> {
+    if (this.initialized) return;
+
+    try {
+      // Create cache directories
+      await fs.mkdir(this.contentPath, { recursive: true });
+      await fs.mkdir(this.metadataPath, { recursive: true });
+
+      // Load or create index
+      this.index = await this.loadIndex();
+
+      // Save index to ensure file exists
+      await this.saveIndex();
+
+      this.initialized = true;
+    } catch (error) {
+      throw new CacheError(
+        `Failed to initialize cache: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        undefined,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * Check if the cache is initialized
+   */
+  isInitialized(): boolean {
+    return this.initialized;
+  }
+
+  /**
+   * Get a cached entry by key
+   */
+  async get<T = string>(key: string, options?: CacheOptions): Promise<CacheEntry<T> | null> {
+    this.ensureInitialized();
+
+    if (options?.skipCache) {
+      return null;
+    }
+
+    const indexEntry = this.index?.entries[key];
+    if (!indexEntry) {
+      return null;
+    }
+
+    // Check expiration (unless force refresh)
+    if (!options?.forceRefresh) {
+      const status = this.getEntryStatus(indexEntry);
+      if (status === 'expired') {
+        return null;
+      }
+    }
+
+    try {
+      // Read metadata
+      const metadataContent = await fs.readFile(indexEntry.metadataFile, 'utf-8');
+      const metadata: CacheMetadata = JSON.parse(metadataContent);
+
+      // Read content
+      const contentRaw = await fs.readFile(indexEntry.contentFile, 'utf-8');
+      let content: T;
+
+      // Try to parse as JSON if content type indicates it
+      if (metadata.contentType.includes('json')) {
+        content = JSON.parse(contentRaw) as T;
+      } else {
+        content = contentRaw as T;
+      }
+
+      return { metadata, content };
+    } catch {
+      // Entry exists in index but files are missing - clean up
+      await this.delete(key);
+      return null;
+    }
+  }
+
+  /**
+   * Get the content only (convenience method)
+   */
+  async getContent<T = string>(key: string, options?: CacheOptions): Promise<T | null> {
+    const entry = await this.get<T>(key, options);
+    return entry?.content ?? null;
+  }
+
+  /**
+   * Set a cache entry
+   */
+  async set<T = string>(
+    key: string,
+    content: T,
+    metadata: Partial<CacheMetadata>,
+    options?: CacheOptions
+  ): Promise<void> {
+    this.ensureInitialized();
+
+    const now = new Date();
+    const ttl = options?.ttl ?? this.ttl;
+    const expiresAt = new Date(now.getTime() + ttl * 1000);
+
+    // Serialize content
+    const contentStr = typeof content === 'string' ? content : JSON.stringify(content, null, 2);
+    const contentHash = this.hashContent(contentStr);
+    const size = Buffer.byteLength(contentStr, 'utf-8');
+
+    // Generate file paths
+    const safeKey = this.sanitizeKey(key);
+    const contentFile = join(this.contentPath, `${safeKey}.cache`);
+    const metadataFile = join(this.metadataPath, `${safeKey}.json`);
+
+    // Build full metadata
+    const fullMetadata: CacheMetadata = {
+      key,
+      source: metadata.source ?? 'custom',
+      sourceUrl: metadata.sourceUrl ?? '',
+      cachedAt: now.toISOString(),
+      expiresAt: expiresAt.toISOString(),
+      contentHash,
+      size,
+      contentType:
+        metadata.contentType ?? (typeof content === 'string' ? 'text/plain' : 'application/json'),
+    };
+    // Conditionally add optional properties
+    if (metadata.etag) {
+      fullMetadata.etag = metadata.etag;
+    }
+    if (metadata.title) {
+      fullMetadata.title = metadata.title;
+    }
+    if (metadata.lastModified) {
+      fullMetadata.lastModified = metadata.lastModified;
+    }
+    const tags = options?.tags ?? metadata.tags;
+    if (tags) {
+      fullMetadata.tags = tags;
+    }
+
+    try {
+      // Ensure directories exist
+      await fs.mkdir(dirname(contentFile), { recursive: true });
+      await fs.mkdir(dirname(metadataFile), { recursive: true });
+
+      // Write files
+      await fs.writeFile(contentFile, contentStr, 'utf-8');
+      await fs.writeFile(metadataFile, JSON.stringify(fullMetadata, null, 2), 'utf-8');
+
+      // Update index
+      if (this.index) {
+        // Remove old size from stats if updating
+        const oldEntry = this.index.entries[key];
+        if (oldEntry) {
+          this.index.stats.totalSize -= oldEntry.size;
+        } else {
+          this.index.stats.entryCount++;
+        }
+
+        this.index.entries[key] = {
+          key,
+          source: fullMetadata.source,
+          cachedAt: fullMetadata.cachedAt,
+          expiresAt: fullMetadata.expiresAt,
+          size,
+          contentFile,
+          metadataFile,
+        };
+        this.index.stats.totalSize += size;
+
+        await this.saveIndex();
+      }
+    } catch (error) {
+      throw new CacheError(
+        `Failed to write cache entry: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        key,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * Check if a key exists in the cache
+   */
+  async has(key: string): Promise<boolean> {
+    this.ensureInitialized();
+    return key in (this.index?.entries ?? {});
+  }
+
+  /**
+   * Delete a cache entry
+   */
+  async delete(key: string): Promise<boolean> {
+    this.ensureInitialized();
+
+    const indexEntry = this.index?.entries[key];
+    if (!indexEntry) {
+      return false;
+    }
+
+    try {
+      // Delete files (ignore if already missing)
+      await fs.unlink(indexEntry.contentFile).catch(noop);
+      await fs.unlink(indexEntry.metadataFile).catch(noop);
+
+      // Update index
+      if (this.index) {
+        this.index.stats.totalSize -= indexEntry.size;
+        this.index.stats.entryCount--;
+        // Remove entry from index using destructuring
+        const { [key]: _removed, ...remaining } = this.index.entries;
+        this.index.entries = remaining;
+        await this.saveIndex();
+      }
+
+      return true;
+    } catch (error) {
+      throw new CacheError(
+        `Failed to delete cache entry: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        key,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * Get the status of a cache entry
+   */
+  async getStatus(key: string): Promise<CacheStatus | null> {
+    this.ensureInitialized();
+
+    const indexEntry = this.index?.entries[key];
+    if (!indexEntry) {
+      return null;
+    }
+
+    return this.getEntryStatus(indexEntry);
+  }
+
+  /**
+   * Get metadata for a cache entry
+   */
+  async getMetadata(key: string): Promise<CacheMetadata | null> {
+    this.ensureInitialized();
+
+    const indexEntry = this.index?.entries[key];
+    if (!indexEntry) {
+      return null;
+    }
+
+    try {
+      const content = await fs.readFile(indexEntry.metadataFile, 'utf-8');
+      return JSON.parse(content);
+    } catch {
+      return null;
+    }
+  }
+
+  /**
+   * Update metadata for a cache entry
+   */
+  async updateMetadata(key: string, metadata: Partial<CacheMetadata>): Promise<void> {
+    this.ensureInitialized();
+
+    const existing = await this.getMetadata(key);
+    if (!existing) {
+      throw new CacheError(`Cache entry not found: ${key}`, 'not_found', key);
+    }
+
+    const indexEntry = this.index?.entries[key];
+    if (!indexEntry) {
+      throw new CacheError(`Cache entry not found: ${key}`, 'not_found', key);
+    }
+
+    const updated: CacheMetadata = {
+      ...existing,
+      ...metadata,
+      key: existing.key, // Can't change key
+    };
+
+    try {
+      await fs.writeFile(indexEntry.metadataFile, JSON.stringify(updated, null, 2), 'utf-8');
+
+      // Update index if relevant fields changed
+      if (this.index && indexEntry) {
+        if (metadata.source) indexEntry.source = metadata.source;
+        if (metadata.expiresAt) indexEntry.expiresAt = metadata.expiresAt;
+        await this.saveIndex();
+      }
+    } catch (error) {
+      throw new CacheError(
+        `Failed to update metadata: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        key,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * List cache entries
+   */
+  async list(options?: CacheListOptions): Promise<CacheMetadata[]> {
+    this.ensureInitialized();
+
+    if (!this.index) return [];
+
+    let entries = Object.values(this.index.entries);
+
+    // Apply filters
+    if (options?.source) {
+      entries = entries.filter((e) => e.source === options.source);
+    }
+
+    if (options?.status) {
+      entries = entries.filter((e) => this.getEntryStatus(e) === options.status);
+    }
+
+    // Load full metadata for tag filtering and return
+    const metadataPromises = entries.map(async (e) => {
+      const metadata = await this.getMetadata(e.key);
+      return metadata;
+    });
+
+    let results = (await Promise.all(metadataPromises)).filter(
+      (m): m is CacheMetadata => m !== null
+    );
+
+    // Filter by tag if specified
+    if (options?.tag) {
+      const tag = options.tag;
+      results = results.filter((m) => m.tags?.includes(tag));
+    }
+
+    // Apply limit
+    if (options?.limit && results.length > options.limit) {
+      results = results.slice(0, options.limit);
+    }
+
+    return results;
+  }
+
+  /**
+   * Get cache statistics
+   */
+  async getStats(): Promise<CacheStats> {
+    this.ensureInitialized();
+
+    if (!this.index) {
+      return {
+        totalEntries: 0,
+        totalSize: 0,
+        validEntries: 0,
+        staleEntries: 0,
+        expiredEntries: 0,
+        bySource: {
+          confluence: 0,
+          github: 0,
+          notion: 0,
+          local: 0,
+          custom: 0,
+        },
+      };
+    }
+
+    const entries = Object.values(this.index.entries);
+    const bySource: Record<CacheSource, number> = {
+      confluence: 0,
+      github: 0,
+      notion: 0,
+      local: 0,
+      custom: 0,
+    };
+
+    let validEntries = 0;
+    let staleEntries = 0;
+    let expiredEntries = 0;
+    let oldestEntry: string | undefined;
+    let newestEntry: string | undefined;
+
+    for (const entry of entries) {
+      bySource[entry.source]++;
+
+      const status = this.getEntryStatus(entry);
+      if (status === 'valid') validEntries++;
+      else if (status === 'stale') staleEntries++;
+      else if (status === 'expired') expiredEntries++;
+
+      if (!oldestEntry || entry.cachedAt < oldestEntry) {
+        oldestEntry = entry.cachedAt;
+      }
+      if (!newestEntry || entry.cachedAt > newestEntry) {
+        newestEntry = entry.cachedAt;
+      }
+    }
+
+    const stats: CacheStats = {
+      totalEntries: this.index.stats.entryCount,
+      totalSize: this.index.stats.totalSize,
+      validEntries,
+      staleEntries,
+      expiredEntries,
+      bySource,
+    };
+
+    if (oldestEntry) stats.oldestEntry = oldestEntry;
+    if (newestEntry) stats.newestEntry = newestEntry;
+
+    return stats;
+  }
+
+  /**
+   * Clear all cache entries
+   */
+  async clear(): Promise<number> {
+    this.ensureInitialized();
+
+    if (!this.index) return 0;
+
+    const count = this.index.stats.entryCount;
+
+    try {
+      // Delete all content and metadata files
+      await fs.rm(this.contentPath, { recursive: true, force: true });
+      await fs.rm(this.metadataPath, { recursive: true, force: true });
+
+      // Recreate directories
+      await fs.mkdir(this.contentPath, { recursive: true });
+      await fs.mkdir(this.metadataPath, { recursive: true });
+
+      // Reset index
+      this.index = this.createEmptyIndex();
+      await this.saveIndex();
+
+      return count;
+    } catch (error) {
+      throw new CacheError(
+        `Failed to clear cache: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        undefined,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  /**
+   * Clear expired entries only
+   */
+  async clearExpired(): Promise<number> {
+    this.ensureInitialized();
+
+    if (!this.index) return 0;
+
+    const expiredKeys = Object.entries(this.index.entries)
+      .filter(([, entry]) => this.getEntryStatus(entry) === 'expired')
+      .map(([key]) => key);
+
+    for (const key of expiredKeys) {
+      await this.delete(key);
+    }
+
+    if (this.index) {
+      this.index.stats.lastCleanup = new Date().toISOString();
+      await this.saveIndex();
+    }
+
+    return expiredKeys.length;
+  }
+
+  /**
+   * Clear entries by source
+   */
+  async clearBySource(source: string): Promise<number> {
+    this.ensureInitialized();
+
+    if (!this.index) return 0;
+
+    const keys = Object.entries(this.index.entries)
+      .filter(([, entry]) => entry.source === source)
+      .map(([key]) => key);
+
+    for (const key of keys) {
+      await this.delete(key);
+    }
+
+    return keys.length;
+  }
+
+  /**
+   * Clear entries by tag
+   */
+  async clearByTag(tag: string): Promise<number> {
+    this.ensureInitialized();
+
+    const entries = await this.list({ tag });
+    for (const entry of entries) {
+      await this.delete(entry.key);
+    }
+
+    return entries.length;
+  }
+
+  // Private helpers
+
+  private ensureInitialized(): void {
+    if (!this.initialized) {
+      throw new CacheError('Cache not initialized. Call initialize() first.', 'invalid_config');
+    }
+  }
+
+  private async loadIndex(): Promise<CacheIndex> {
+    try {
+      const content = await fs.readFile(this.indexPath, 'utf-8');
+      const index = JSON.parse(content) as CacheIndex;
+
+      // Validate version
+      if (index.version !== INDEX_VERSION) {
+        // Index version mismatch - rebuild
+        return this.rebuildIndex();
+      }
+
+      return index;
+    } catch {
+      // Index doesn't exist or is invalid - create new one
+      return this.createEmptyIndex();
+    }
+  }
+
+  private async saveIndex(): Promise<void> {
+    if (!this.index) return;
+
+    try {
+      await fs.writeFile(this.indexPath, JSON.stringify(this.index, null, 2), 'utf-8');
+    } catch (error) {
+      throw new CacheError(
+        `Failed to save cache index: ${error instanceof Error ? error.message : String(error)}`,
+        'write_failed',
+        undefined,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  private createEmptyIndex(): CacheIndex {
+    return {
+      version: INDEX_VERSION,
+      entries: {},
+      stats: {
+        totalSize: 0,
+        entryCount: 0,
+        lastCleanup: null,
+      },
+    };
+  }
+
+  private async rebuildIndex(): Promise<CacheIndex> {
+    const index = this.createEmptyIndex();
+
+    try {
+      const metadataFiles = await fs.readdir(this.metadataPath);
+
+      for (const file of metadataFiles) {
+        if (!file.endsWith('.json')) continue;
+
+        try {
+          const metadataPath = join(this.metadataPath, file);
+          const content = await fs.readFile(metadataPath, 'utf-8');
+          const metadata: CacheMetadata = JSON.parse(content);
+
+          const safeKey = this.sanitizeKey(metadata.key);
+          const contentFile = join(this.contentPath, `${safeKey}.cache`);
+
+          // Verify content file exists
+          await fs.access(contentFile);
+
+          index.entries[metadata.key] = {
+            key: metadata.key,
+            source: metadata.source,
+            cachedAt: metadata.cachedAt,
+            expiresAt: metadata.expiresAt,
+            size: metadata.size,
+            contentFile,
+            metadataFile: metadataPath,
+          };
+
+          index.stats.totalSize += metadata.size;
+          index.stats.entryCount++;
+        } catch {
+          // Skip invalid entries
+        }
+      }
+    } catch {
+      // Metadata directory doesn't exist or is empty
+    }
+
+    return index;
+  }
+
+  private getEntryStatus(entry: CacheIndexEntry): CacheStatus {
+    const now = new Date();
+    const expiresAt = new Date(entry.expiresAt);
+    const cachedAt = new Date(entry.cachedAt);
+
+    if (now > expiresAt) {
+      return 'expired';
+    }
+
+    // Consider stale if more than 80% of TTL has passed
+    const totalTtl = expiresAt.getTime() - cachedAt.getTime();
+    const elapsed = now.getTime() - cachedAt.getTime();
+    if (elapsed > totalTtl * 0.8) {
+      return 'stale';
+    }
+
+    return 'valid';
+  }
+
+  private sanitizeKey(key: string): string {
+    // Create a safe filename from the key
+    return createHash('sha256').update(key).digest('hex').substring(0, 32);
+  }
+
+  private hashContent(content: string): string {
+    return createHash('sha256').update(content).digest('hex');
+  }
+}
+
+/**
+ * Create a file cache provider
+ */
+export function createFileCacheProvider(options: FileCacheProviderOptions): FileCacheProvider {
+  return new FileCacheProvider(options);
+}