@deimoscloud/coreai 0.1.0

package/src/cache/manager.ts

@@ -0,0 +1,388 @@
+/**
+ * Cache Manager
+ *
+ * Higher-level cache management that combines caching with remote fetching.
+ * Implements cache-first resolution with remote fallback strategy.
+ */
+
+import type { CacheEntry, CacheMetadata, CacheOptions, CacheSource, SyncResult } from './types.js';
+import { CacheError } from './types.js';
+import type {
+  CacheProvider,
+  CacheManager as ICacheManager,
+  RemoteFetcher,
+  FetchOptions,
+  SyncOptions,
+} from './interfaces.js';
+
+/**
+ * Options for creating a cache manager
+ */
+export interface CacheManagerOptions {
+  /**
+   * The underlying cache provider
+   */
+  cache: CacheProvider;
+
+  /**
+   * Default TTL in seconds
+   */
+  defaultTtl?: number;
+
+  /**
+   * Default source for new entries
+   */
+  defaultSource?: CacheSource;
+}
+
+/**
+ * Cache manager implementation
+ *
+ * Provides cache-first resolution strategy:
+ * 1. Check cache for valid entry
+ * 2. If cached and valid, return cached content
+ * 3. If stale or missing, fetch from remote
+ * 4. Cache the fetched content
+ * 5. Return the content
+ */
+export class CacheManager implements ICacheManager {
+  private cache: CacheProvider;
+  private fetchers = new Map<string, RemoteFetcher>();
+  private defaultTtl: number;
+  private defaultSource: CacheSource;
+
+  constructor(options: CacheManagerOptions) {
+    this.cache = options.cache;
+    this.defaultTtl = options.defaultTtl ?? 3600;
+    this.defaultSource = options.defaultSource ?? 'custom';
+  }
+
+  /**
+   * Get content with cache-first strategy
+   */
+  async getWithFallback<T = string>(
+    key: string,
+    url: string,
+    options?: CacheOptions & FetchOptions
+  ): Promise<CacheEntry<T>> {
+    // Try cache first (unless skipCache is set)
+    if (!options?.skipCache && !options?.forceRefresh) {
+      const cached = await this.cache.get<T>(key);
+      if (cached) {
+        const status = await this.cache.getStatus(key);
+        // Return if valid (not stale or expired)
+        if (status === 'valid') {
+          return cached;
+        }
+        // If stale, try to refresh but fall back to stale if fetch fails
+        if (status === 'stale') {
+          try {
+            return await this.fetchAndCache<T>(key, url, options);
+          } catch {
+            // Return stale content on fetch failure
+            return cached;
+          }
+        }
+      }
+    }
+
+    // Cache miss or forceRefresh - fetch from remote
+    return this.fetchAndCache<T>(key, url, options);
+  }
+
+  /**
+   * Sync all entries from a source
+   */
+  async syncSource(source: string, options?: SyncOptions): Promise<SyncResult> {
+    const startTime = Date.now();
+    const result: SyncResult = {
+      added: 0,
+      updated: 0,
+      removed: 0,
+      failed: 0,
+      errors: [],
+      duration: 0,
+    };
+
+    // Get all entries from this source
+    const entries = await this.cache.list({ source: source as CacheSource });
+
+    // Sync each entry
+    const concurrency = options?.concurrency ?? 5;
+    const chunks = this.chunkArray(entries, concurrency);
+
+    for (const chunk of chunks) {
+      const promises = chunk.map(async (entry) => {
+        try {
+          const fetcher = this.fetchers.get(source);
+          if (!fetcher) {
+            throw new Error(`No fetcher registered for source: ${source}`);
+          }
+
+          // Check if content has changed
+          const hasChanged = await fetcher.hasChanged(entry.sourceUrl, entry.etag);
+
+          if (hasChanged || options?.force) {
+            // Fetch and update
+            const fetchOptions: FetchOptions = {};
+            if (entry.etag) {
+              fetchOptions.etag = entry.etag;
+            }
+            const { content, metadata } = await fetcher.fetch(entry.sourceUrl, fetchOptions);
+
+            await this.cache.set(entry.key, content, {
+              ...entry,
+              ...metadata,
+            });
+
+            result.updated++;
+          }
+        } catch (error) {
+          result.failed++;
+          result.errors.push({
+            key: entry.key,
+            error: error instanceof Error ? error.message : String(error),
+          });
+
+          if (!options?.continueOnError) {
+            throw error;
+          }
+        }
+      });
+
+      await Promise.all(promises);
+    }
+
+    // Report final progress
+    if (options?.onProgress) {
+      options.onProgress(entries.length, entries.length);
+    }
+
+    result.duration = Date.now() - startTime;
+    return result;
+  }
+
+  /**
+   * Sync specific entries
+   */
+  async syncEntries(keys: string[], options?: SyncOptions): Promise<SyncResult> {
+    const startTime = Date.now();
+    const result: SyncResult = {
+      added: 0,
+      updated: 0,
+      removed: 0,
+      failed: 0,
+      errors: [],
+      duration: 0,
+    };
+
+    const concurrency = options?.concurrency ?? 5;
+    const chunks = this.chunkArray(keys, concurrency);
+
+    for (const chunk of chunks) {
+      const promises = chunk.map(async (key) => {
+        try {
+          const metadata = await this.cache.getMetadata(key);
+          if (!metadata) {
+            result.failed++;
+            result.errors.push({ key, error: 'Entry not found in cache' });
+            return;
+          }
+
+          const fetcher = this.fetchers.get(metadata.source);
+          if (!fetcher) {
+            result.failed++;
+            result.errors.push({ key, error: `No fetcher for source: ${metadata.source}` });
+            return;
+          }
+
+          // Check if content has changed
+          const hasChanged = await fetcher.hasChanged(metadata.sourceUrl, metadata.etag);
+
+          if (hasChanged || options?.force) {
+            const fetchOpts: FetchOptions = {};
+            if (metadata.etag) {
+              fetchOpts.etag = metadata.etag;
+            }
+            const { content, metadata: newMeta } = await fetcher.fetch(
+              metadata.sourceUrl,
+              fetchOpts
+            );
+
+            await this.cache.set(key, content, {
+              ...metadata,
+              ...newMeta,
+            });
+
+            result.updated++;
+          }
+        } catch (error) {
+          result.failed++;
+          result.errors.push({
+            key,
+            error: error instanceof Error ? error.message : String(error),
+          });
+
+          if (!options?.continueOnError) {
+            throw error;
+          }
+        }
+      });
+
+      await Promise.all(promises);
+    }
+
+    // Report final progress
+    if (options?.onProgress) {
+      options.onProgress(keys.length, keys.length);
+    }
+
+    result.duration = Date.now() - startTime;
+    return result;
+  }
+
+  /**
+   * Register a fetcher for a source
+   */
+  registerFetcher(source: string, fetcher: RemoteFetcher): void {
+    this.fetchers.set(source, fetcher);
+  }
+
+  /**
+   * Get the underlying cache provider
+   */
+  getCache(): CacheProvider {
+    return this.cache;
+  }
+
+  /**
+   * Check if a fetcher is registered for a source
+   */
+  hasFetcher(source: string): boolean {
+    return this.fetchers.has(source);
+  }
+
+  /**
+   * Get all registered sources
+   */
+  getRegisteredSources(): string[] {
+    return Array.from(this.fetchers.keys());
+  }
+
+  // Private helpers
+
+  private async fetchAndCache<T>(
+    key: string,
+    url: string,
+    options?: CacheOptions & FetchOptions
+  ): Promise<CacheEntry<T>> {
+    // Determine source from URL
+    const source = this.getSourceFromUrl(url);
+    const fetcher = this.fetchers.get(source);
+
+    if (!fetcher) {
+      throw new CacheError(
+        `No fetcher registered for source: ${source}. Register one with registerFetcher().`,
+        'fetch_failed',
+        key
+      );
+    }
+
+    try {
+      // Fetch from remote
+      const fetchOpts: FetchOptions = {};
+      if (options?.timeout) {
+        fetchOpts.timeout = options.timeout;
+      }
+      if (options?.headers) {
+        fetchOpts.headers = options.headers;
+      }
+      const { content, metadata } = await fetcher.fetch(url, fetchOpts);
+
+      // Build full metadata
+      const fullMetadata: Partial<CacheMetadata> = {
+        source: source as CacheSource,
+        sourceUrl: url,
+        contentType: metadata.contentType ?? 'text/plain',
+      };
+      if (metadata.etag) {
+        fullMetadata.etag = metadata.etag;
+      }
+      if (metadata.title) {
+        fullMetadata.title = metadata.title;
+      }
+      if (metadata.lastModified) {
+        fullMetadata.lastModified = metadata.lastModified;
+      }
+      if (options?.tags) {
+        fullMetadata.tags = options.tags;
+      }
+
+      // Cache the content
+      const cacheOpts: CacheOptions = {
+        ttl: options?.ttl ?? this.defaultTtl,
+      };
+      if (options?.tags) {
+        cacheOpts.tags = options.tags;
+      }
+      await this.cache.set(key, content, fullMetadata, cacheOpts);
+
+      // Return the entry
+      const entry = await this.cache.get<T>(key);
+      if (!entry) {
+        throw new CacheError('Failed to retrieve cached entry after write', 'read_failed', key);
+      }
+
+      return entry;
+    } catch (error) {
+      if (error instanceof CacheError) {
+        throw error;
+      }
+      throw new CacheError(
+        `Failed to fetch content: ${error instanceof Error ? error.message : String(error)}`,
+        'fetch_failed',
+        key,
+        error instanceof Error ? error : undefined
+      );
+    }
+  }
+
+  private getSourceFromUrl(url: string): string {
+    try {
+      const parsed = new URL(url);
+      const hostname = parsed.hostname.toLowerCase();
+
+      // Map common hostnames to sources
+      if (hostname.includes('confluence') || hostname.includes('atlassian.net')) {
+        return 'confluence';
+      }
+      if (hostname.includes('github.com') || hostname.includes('github')) {
+        return 'github';
+      }
+      if (hostname.includes('notion.so') || hostname.includes('notion')) {
+        return 'notion';
+      }
+
+      // Default to custom
+      return this.defaultSource;
+    } catch {
+      // Invalid URL, might be a file path
+      return 'local';
+    }
+  }
+
+  private chunkArray<T>(array: T[], size: number): T[][] {
+    const chunks: T[][] = [];
+    for (let i = 0; i < array.length; i += size) {
+      chunks.push(array.slice(i, i + size));
+    }
+    return chunks;
+  }
+}
+
+/**
+ * Create a cache manager
+ */
+export function createCacheManager(options: CacheManagerOptions): CacheManager {
+  return new CacheManager(options);
+}