@objectstack/metadata 0.7.1

package/src/loaders/filesystem-loader.ts

@@ -0,0 +1,314 @@
+/**
+ * Filesystem Metadata Loader
+ * 
+ * Loads metadata from the filesystem using glob patterns
+ */
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { glob } from 'glob';
+import { createHash } from 'node:crypto';
+import type {
+  MetadataLoadOptions,
+  MetadataLoadResult,
+  MetadataStats,
+  MetadataLoaderContract,
+  MetadataFormat,
+} from '@objectstack/spec/system';
+import type { Logger } from '@objectstack/core';
+import type { MetadataLoader } from './loader-interface.js';
+import type { MetadataSerializer } from '../serializers/serializer-interface.js';
+
+export class FilesystemLoader implements MetadataLoader {
+  readonly contract: MetadataLoaderContract = {
+    name: 'filesystem',
+    supportedFormats: ['json', 'yaml', 'typescript', 'javascript'],
+    supportsWatch: true,
+    supportsWrite: true,
+    supportsCache: true,
+  };
+
+  private cache = new Map<string, { data: any; etag: string; timestamp: number }>();
+
+  constructor(
+    private rootDir: string,
+    private serializers: Map<MetadataFormat, MetadataSerializer>,
+    private logger?: Logger
+  ) {}
+
+  async load(
+    type: string,
+    name: string,
+    options?: MetadataLoadOptions
+  ): Promise<MetadataLoadResult> {
+    const startTime = Date.now();
+    const { validate: _validate = true, useCache = true, ifNoneMatch } = options || {};
+
+    try {
+      // Find the file
+      const filePath = await this.findFile(type, name);
+
+      if (!filePath) {
+        return {
+          data: null,
+          fromCache: false,
+          notModified: false,
+          loadTime: Date.now() - startTime,
+        };
+      }
+
+      // Get stats
+      const stats = await this.stat(type, name);
+
+      if (!stats) {
+        return {
+          data: null,
+          fromCache: false,
+          notModified: false,
+          loadTime: Date.now() - startTime,
+        };
+      }
+
+      // Check cache
+      if (useCache && ifNoneMatch && stats.etag === ifNoneMatch) {
+        return {
+          data: null,
+          fromCache: true,
+          notModified: true,
+          etag: stats.etag,
+          stats,
+          loadTime: Date.now() - startTime,
+        };
+      }
+
+      // Check memory cache
+      const cacheKey = `${type}:${name}`;
+      if (useCache && this.cache.has(cacheKey)) {
+        const cached = this.cache.get(cacheKey)!;
+        if (cached.etag === stats.etag) {
+          return {
+            data: cached.data,
+            fromCache: true,
+            notModified: false,
+            etag: stats.etag,
+            stats,
+            loadTime: Date.now() - startTime,
+          };
+        }
+      }
+
+      // Load and deserialize
+      const content = await fs.readFile(filePath, 'utf-8');
+      const serializer = this.getSerializer(stats.format);
+
+      if (!serializer) {
+        throw new Error(`No serializer found for format: ${stats.format}`);
+      }
+
+      const data = serializer.deserialize(content);
+
+      // Update cache
+      if (useCache) {
+        this.cache.set(cacheKey, {
+          data,
+          etag: stats.etag,
+          timestamp: Date.now(),
+        });
+      }
+
+      return {
+        data,
+        fromCache: false,
+        notModified: false,
+        etag: stats.etag,
+        stats,
+        loadTime: Date.now() - startTime,
+      };
+    } catch (error) {
+      this.logger?.error('Failed to load metadata', undefined, {
+        type,
+        name,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async loadMany<T = any>(
+    type: string,
+    options?: MetadataLoadOptions
+  ): Promise<T[]> {
+    const { patterns = ['**/*'], recursive: _recursive = true, limit } = options || {};
+
+    const typeDir = path.join(this.rootDir, type);
+    const items: T[] = [];
+
+    try {
+      // Build glob patterns
+      const globPatterns = patterns.map(pattern =>
+        path.join(typeDir, pattern)
+      );
+
+      for (const pattern of globPatterns) {
+        const files = await glob(pattern, {
+          ignore: ['**/node_modules/**', '**/*.test.*', '**/*.spec.*'],
+          nodir: true,
+        });
+
+        for (const file of files) {
+          if (limit && items.length >= limit) {
+            break;
+          }
+
+          try {
+            const content = await fs.readFile(file, 'utf-8');
+            const format = this.detectFormat(file);
+            const serializer = this.getSerializer(format);
+
+            if (serializer) {
+              const data = serializer.deserialize<T>(content);
+              items.push(data);
+            }
+          } catch (error) {
+            this.logger?.warn('Failed to load file', {
+              file,
+              error: error instanceof Error ? error.message : String(error),
+            });
+          }
+        }
+
+        if (limit && items.length >= limit) {
+          break;
+        }
+      }
+
+      return items;
+    } catch (error) {
+      this.logger?.error('Failed to load many', undefined, {
+        type,
+        patterns,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async exists(type: string, name: string): Promise<boolean> {
+    const filePath = await this.findFile(type, name);
+    return filePath !== null;
+  }
+
+  async stat(type: string, name: string): Promise<MetadataStats | null> {
+    const filePath = await this.findFile(type, name);
+
+    if (!filePath) {
+      return null;
+    }
+
+    try {
+      const stats = await fs.stat(filePath);
+      const content = await fs.readFile(filePath, 'utf-8');
+      const etag = this.generateETag(content);
+      const format = this.detectFormat(filePath);
+
+      return {
+        size: stats.size,
+        modifiedAt: stats.mtime,
+        etag,
+        format,
+        path: filePath,
+      };
+    } catch (error) {
+      this.logger?.error('Failed to stat file', undefined, {
+        type,
+        name,
+        filePath,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return null;
+    }
+  }
+
+  async list(type: string): Promise<string[]> {
+    const typeDir = path.join(this.rootDir, type);
+
+    try {
+      const files = await glob('**/*', {
+        cwd: typeDir,
+        ignore: ['**/node_modules/**', '**/*.test.*', '**/*.spec.*'],
+        nodir: true,
+      });
+
+      return files.map(file => {
+        const ext = path.extname(file);
+        const basename = path.basename(file, ext);
+        return basename;
+      });
+    } catch (error) {
+      this.logger?.error('Failed to list', undefined, {
+        type,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      return [];
+    }
+  }
+
+  /**
+   * Find file for a given type and name
+   */
+  private async findFile(type: string, name: string): Promise<string | null> {
+    const typeDir = path.join(this.rootDir, type);
+    const extensions = ['.json', '.yaml', '.yml', '.ts', '.js'];
+
+    for (const ext of extensions) {
+      const filePath = path.join(typeDir, `${name}${ext}`);
+
+      try {
+        await fs.access(filePath);
+        return filePath;
+      } catch {
+        // File doesn't exist, try next extension
+      }
+    }
+
+    return null;
+  }
+
+  /**
+   * Detect format from file extension
+   */
+  private detectFormat(filePath: string): MetadataFormat {
+    const ext = path.extname(filePath).toLowerCase();
+
+    switch (ext) {
+      case '.json':
+        return 'json';
+      case '.yaml':
+      case '.yml':
+        return 'yaml';
+      case '.ts':
+        return 'typescript';
+      case '.js':
+        return 'javascript';
+      default:
+        return 'json'; // Default to JSON
+    }
+  }
+
+  /**
+   * Get serializer for format
+   */
+  private getSerializer(format: MetadataFormat): MetadataSerializer | undefined {
+    return this.serializers.get(format);
+  }
+
+  /**
+   * Generate ETag for content
+   * Uses SHA-256 hash truncated to 32 characters for reasonable collision resistance
+   * while keeping ETag headers compact (full 64-char hash is overkill for this use case)
+   */
+  private generateETag(content: string): string {
+    const hash = createHash('sha256').update(content).digest('hex').substring(0, 32);
+    return `"${hash}"`;
+  }
+}

package/src/loaders/loader-interface.ts

@@ -0,0 +1,70 @@
+/**
+ * Metadata Loader Interface
+ * 
+ * Defines the contract for loading metadata from various sources
+ */
+
+import type {
+  MetadataLoadOptions,
+  MetadataLoadResult,
+  MetadataStats,
+  MetadataLoaderContract,
+} from '@objectstack/spec/system';
+
+/**
+ * Abstract interface for metadata loaders
+ * Implementations can load from filesystem, HTTP, S3, databases, etc.
+ */
+export interface MetadataLoader {
+  /**
+   * Loader contract information
+   */
+  readonly contract: MetadataLoaderContract;
+
+  /**
+   * Load a single metadata item
+   * @param type The metadata type (e.g., 'object', 'view', 'app')
+   * @param name The item name/identifier
+   * @param options Load options
+   * @returns Load result with data or null if not found
+   */
+  load(
+    type: string,
+    name: string,
+    options?: MetadataLoadOptions
+  ): Promise<MetadataLoadResult>;
+
+  /**
+   * Load multiple items matching patterns
+   * @param type The metadata type
+   * @param options Load options with patterns
+   * @returns Array of loaded items
+   */
+  loadMany<T = any>(
+    type: string,
+    options?: MetadataLoadOptions
+  ): Promise<T[]>;
+
+  /**
+   * Check if item exists
+   * @param type The metadata type
+   * @param name The item name
+   * @returns True if exists
+   */
+  exists(type: string, name: string): Promise<boolean>;
+
+  /**
+   * Get item metadata (without loading full content)
+   * @param type The metadata type
+   * @param name The item name
+   * @returns Metadata statistics
+   */
+  stat(type: string, name: string): Promise<MetadataStats | null>;
+
+  /**
+   * List all items of a type
+   * @param type The metadata type
+   * @returns Array of item names
+   */
+  list(type: string): Promise<string[]>;
+}

package/src/metadata-manager.ts

@@ -0,0 +1,338 @@
+/**
+ * Metadata Manager
+ * 
+ * Main orchestrator for metadata loading, saving, and persistence
+ */
+
+import * as fs from 'node:fs/promises';
+import * as path from 'node:path';
+import { createHash } from 'node:crypto';
+import { watch as chokidarWatch, type FSWatcher } from 'chokidar';
+import type {
+  MetadataManagerConfig,
+  MetadataLoadOptions,
+  MetadataSaveOptions,
+  MetadataSaveResult,
+  MetadataWatchEvent,
+  MetadataFormat,
+} from '@objectstack/spec/system';
+import { createLogger, type Logger } from '@objectstack/core';
+import { FilesystemLoader } from './loaders/filesystem-loader.js';
+import { JSONSerializer } from './serializers/json-serializer.js';
+import { YAMLSerializer } from './serializers/yaml-serializer.js';
+import { TypeScriptSerializer } from './serializers/typescript-serializer.js';
+import type { MetadataSerializer } from './serializers/serializer-interface.js';
+import type { MetadataLoader } from './loaders/loader-interface.js';
+
+/**
+ * Watch callback function
+ */
+export type WatchCallback = (event: MetadataWatchEvent) => void | Promise<void>;
+
+/**
+ * Main metadata manager class
+ */
+export class MetadataManager {
+  private loader: MetadataLoader;
+  private serializers: Map<MetadataFormat, MetadataSerializer>;
+  private logger: Logger;
+  private watcher?: FSWatcher;
+  private watchCallbacks = new Map<string, Set<WatchCallback>>();
+
+  constructor(private config: MetadataManagerConfig) {
+    this.logger = createLogger({ level: 'info', format: 'pretty' });
+
+    // Initialize serializers
+    this.serializers = new Map();
+    const formats = config.formats || ['typescript', 'json', 'yaml'];
+
+    if (formats.includes('json')) {
+      this.serializers.set('json', new JSONSerializer());
+    }
+    if (formats.includes('yaml')) {
+      this.serializers.set('yaml', new YAMLSerializer());
+    }
+    if (formats.includes('typescript')) {
+      this.serializers.set('typescript', new TypeScriptSerializer('typescript'));
+    }
+    if (formats.includes('javascript')) {
+      this.serializers.set('javascript', new TypeScriptSerializer('javascript'));
+    }
+
+    // Initialize loader
+    const rootDir = config.rootDir || process.cwd();
+    this.loader = new FilesystemLoader(rootDir, this.serializers, this.logger);
+
+    // Start watching if enabled
+    if (config.watch) {
+      this.startWatching();
+    }
+  }
+
+  /**
+   * Load a single metadata item
+   */
+  async load<T = any>(
+    type: string,
+    name: string,
+    options?: MetadataLoadOptions
+  ): Promise<T | null> {
+    const result = await this.loader.load(type, name, options);
+    return result.data;
+  }
+
+  /**
+   * Load multiple metadata items
+   */
+  async loadMany<T = any>(
+    type: string,
+    options?: MetadataLoadOptions
+  ): Promise<T[]> {
+    return this.loader.loadMany<T>(type, options);
+  }
+
+  /**
+   * Save metadata to disk
+   */
+  async save<T = any>(
+    type: string,
+    name: string,
+    data: T,
+    options?: MetadataSaveOptions
+  ): Promise<MetadataSaveResult> {
+    const startTime = Date.now();
+    const {
+      format = 'typescript',
+      prettify = true,
+      indent = 2,
+      sortKeys = false,
+      backup = false,
+      overwrite = true,
+      atomic = true,
+      path: customPath,
+    } = options || {};
+
+    try {
+      // Get serializer
+      const serializer = this.serializers.get(format);
+      if (!serializer) {
+        throw new Error(`No serializer found for format: ${format}`);
+      }
+
+      // Determine file path
+      const typeDir = path.join(this.config.rootDir || process.cwd(), type);
+      const fileName = `${name}${serializer.getExtension()}`;
+      const filePath = customPath || path.join(typeDir, fileName);
+
+      // Check if file exists
+      if (!overwrite) {
+        try {
+          await fs.access(filePath);
+          throw new Error(`File already exists: ${filePath}`);
+        } catch (error) {
+          // File doesn't exist, continue
+          if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+            throw error;
+          }
+        }
+      }
+
+      // Create directory if it doesn't exist
+      await fs.mkdir(path.dirname(filePath), { recursive: true });
+
+      // Create backup if requested
+      let backupPath: string | undefined;
+      if (backup) {
+        try {
+          await fs.access(filePath);
+          backupPath = `${filePath}.bak`;
+          await fs.copyFile(filePath, backupPath);
+        } catch {
+          // File doesn't exist, no backup needed
+        }
+      }
+
+      // Serialize data
+      const content = serializer.serialize(data, {
+        prettify,
+        indent,
+        sortKeys,
+      });
+
+      // Write to disk (atomic or direct)
+      if (atomic) {
+        const tempPath = `${filePath}.tmp`;
+        await fs.writeFile(tempPath, content, 'utf-8');
+        await fs.rename(tempPath, filePath);
+      } else {
+        await fs.writeFile(filePath, content, 'utf-8');
+      }
+
+      // Get stats
+      const stats = await fs.stat(filePath);
+      const etag = this.generateETag(content);
+
+      return {
+        success: true,
+        path: filePath,
+        etag,
+        size: stats.size,
+        saveTime: Date.now() - startTime,
+        backupPath,
+      };
+    } catch (error) {
+      this.logger.error('Failed to save metadata', undefined, {
+        type,
+        name,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  /**
+   * Check if metadata item exists
+   */
+  async exists(type: string, name: string): Promise<boolean> {
+    return this.loader.exists(type, name);
+  }
+
+  /**
+   * List all items of a type
+   */
+  async list(type: string): Promise<string[]> {
+    return this.loader.list(type);
+  }
+
+  /**
+   * Watch for metadata changes
+   */
+  watch(type: string, callback: WatchCallback): void {
+    if (!this.watchCallbacks.has(type)) {
+      this.watchCallbacks.set(type, new Set());
+    }
+    this.watchCallbacks.get(type)!.add(callback);
+  }
+
+  /**
+   * Unwatch metadata changes
+   */
+  unwatch(type: string, callback: WatchCallback): void {
+    const callbacks = this.watchCallbacks.get(type);
+    if (callbacks) {
+      callbacks.delete(callback);
+      if (callbacks.size === 0) {
+        this.watchCallbacks.delete(type);
+      }
+    }
+  }
+
+  /**
+   * Stop all watching
+   */
+  async stopWatching(): Promise<void> {
+    if (this.watcher) {
+      await this.watcher.close();
+      this.watcher = undefined;
+      this.watchCallbacks.clear();
+    }
+  }
+
+  /**
+   * Start watching for file changes
+   */
+  private startWatching(): void {
+    const rootDir = this.config.rootDir || process.cwd();
+    const { ignored = ['**/node_modules/**', '**/*.test.*'], persistent = true } =
+      this.config.watchOptions || {};
+
+    this.watcher = chokidarWatch(rootDir, {
+      ignored,
+      persistent,
+      ignoreInitial: true,
+    });
+
+    this.watcher.on('add', async (filePath) => {
+      await this.handleFileEvent('added', filePath);
+    });
+
+    this.watcher.on('change', async (filePath) => {
+      await this.handleFileEvent('changed', filePath);
+    });
+
+    this.watcher.on('unlink', async (filePath) => {
+      await this.handleFileEvent('deleted', filePath);
+    });
+
+    this.logger.info('File watcher started', { rootDir });
+  }
+
+  /**
+   * Handle file change events
+   */
+  private async handleFileEvent(
+    eventType: 'added' | 'changed' | 'deleted',
+    filePath: string
+  ): Promise<void> {
+    const rootDir = this.config.rootDir || process.cwd();
+    const relativePath = path.relative(rootDir, filePath);
+    const parts = relativePath.split(path.sep);
+
+    if (parts.length < 2) {
+      return; // Not a metadata file
+    }
+
+    const type = parts[0];
+    const fileName = parts[parts.length - 1];
+    const name = path.basename(fileName, path.extname(fileName));
+
+    const callbacks = this.watchCallbacks.get(type);
+    if (!callbacks || callbacks.size === 0) {
+      return;
+    }
+
+    let data: any = undefined;
+    if (eventType !== 'deleted') {
+      try {
+        data = await this.load(type, name, { useCache: false });
+      } catch (error) {
+        this.logger.error('Failed to load changed file', undefined, {
+          filePath,
+          error: error instanceof Error ? error.message : String(error),
+        });
+        return;
+      }
+    }
+
+    const event: MetadataWatchEvent = {
+      type: eventType,
+      metadataType: type,
+      name,
+      path: filePath,
+      data,
+      timestamp: new Date(),
+    };
+
+    for (const callback of callbacks) {
+      try {
+        await callback(event);
+      } catch (error) {
+        this.logger.error('Watch callback error', undefined, {
+          type,
+          name,
+          error: error instanceof Error ? error.message : String(error),
+        });
+      }
+    }
+  }
+
+  /**
+   * Generate ETag for content
+   * Uses SHA-256 hash truncated to 32 characters for reasonable collision resistance
+   * while keeping ETag headers compact (full 64-char hash is overkill for this use case)
+   */
+  private generateETag(content: string): string {
+    const hash = createHash('sha256').update(content).digest('hex').substring(0, 32);
+    return `"${hash}"`;
+  }
+}