clearctx 3.0.0

package/src/artifact-store.js

@@ -0,0 +1,710 @@
+/**
+ * artifact-store.js
+ * Layer 2: Artifact Store with Versioning, Immutability, and Schema Validation
+ *
+ * This module provides a versioned artifact storage system that ensures:
+ * - Immutability: Once published, artifact versions cannot be changed
+ * - Schema validation: Data is validated against JSON schemas before storage
+ * - Version tracking: Each artifact can have multiple versions
+ * - Metadata: Rich metadata including tags, lineage, checksums
+ */
+
+const fs = require('fs');
+const path = require('path');
+const os = require('os');
+const crypto = require('crypto');
+
+// Import Layer 1 utilities (being built in parallel)
+const { atomicWriteJson, writeImmutable, readJsonSafe } = require('./atomic-io');
+const { acquireLock, releaseLock } = require('./file-lock');
+
+/**
+ * ArtifactStore manages versioned, immutable artifacts with schema validation
+ *
+ * Directory structure:
+ * team/{teamName}/artifacts/
+ *   index.json          - artifact registry (locked for writes)
+ *   schemas/            - JSON schema files for well-known types
+ *   data/{artifactId}/  - one dir per artifact
+ *     v1.json, v2.json  - immutable version files
+ */
+class ArtifactStore {
+  /**
+   * Create a new ArtifactStore instance
+   * @param {string} teamName - Name of the team (default: 'default')
+   */
+  constructor(teamName = 'default') {
+    // Set up base directory structure
+    const baseDir = path.join(os.homedir(), '.clearctx');
+    const teamDir = path.join(baseDir, 'team', teamName);
+
+    // Define all directory paths
+    this.artifactsDir = path.join(teamDir, 'artifacts');
+    this.indexPath = path.join(this.artifactsDir, 'index.json');
+    this.schemasDir = path.join(this.artifactsDir, 'schemas');
+    this.dataDir = path.join(this.artifactsDir, 'data');
+    this.locksDir = path.join(teamDir, 'locks');
+
+    // Create all necessary directories
+    this._ensureDirectories();
+
+    // Create default schemas if they don't exist
+    this._createDefaultSchemas();
+  }
+
+  /**
+   * Ensure all required directories exist
+   * @private
+   */
+  _ensureDirectories() {
+    // Create directories recursively if they don't exist
+    [this.artifactsDir, this.schemasDir, this.dataDir, this.locksDir].forEach(dir => {
+      if (!fs.existsSync(dir)) {
+        fs.mkdirSync(dir, { recursive: true });
+      }
+    });
+  }
+
+  /**
+   * Load a JSON schema for a specific artifact type
+   * @param {string} type - The artifact type
+   * @returns {Object|null} The schema object, or null if not found
+   * @private
+   */
+  _loadSchema(type) {
+    const schemaPath = path.join(this.schemasDir, `${type}.json`);
+
+    // Try to read the schema file
+    try {
+      if (!fs.existsSync(schemaPath)) {
+        return null; // Custom types have no schema
+      }
+
+      const schemaContent = fs.readFileSync(schemaPath, 'utf8');
+      return JSON.parse(schemaContent);
+    } catch (err) {
+      // If schema file is corrupted or invalid, return null
+      return null;
+    }
+  }
+
+  /**
+   * Validate data against a JSON schema
+   * Implements basic JSON Schema validation (required fields, types, enums)
+   * @param {string} type - The artifact type
+   * @param {Object} data - The data to validate
+   * @returns {Object} { valid: true } or { valid: false, errors: [...] }
+   * @private
+   */
+  _validateData(type, data) {
+    // Load the schema for this type
+    const schema = this._loadSchema(type);
+
+    // If no schema exists, validation passes (custom types)
+    if (!schema) {
+      return { valid: true };
+    }
+
+    const errors = [];
+
+    // Check required fields
+    if (schema.required && Array.isArray(schema.required)) {
+      for (const field of schema.required) {
+        if (!(field in data)) {
+          // Build helpful error message with type-specific suggestions
+          let errorMsg = `field '${field}' is required`;
+
+          // Add helpful suggestions for well-known types
+          const typeHints = {
+            'api-contract': `type 'api-contract' requires an 'endpoints' field. If this artifact isn't an API contract, use type 'custom' instead.`,
+            'schema-change': `type 'schema-change' requires a 'models' field. If this artifact isn't a database schema change, use type 'custom' instead.`,
+            'test-results': `type 'test-results' requires 'total', 'passed', and 'failed' fields. If this artifact isn't test results, use type 'custom' instead.`,
+            'component-spec': `type 'component-spec' requires a 'componentName' field. If this artifact isn't a component specification, use type 'custom' instead.`,
+            'file-manifest': `type 'file-manifest' requires a 'files' field. If this artifact isn't a file manifest, use type 'custom' instead.`,
+            'config-change': `type 'config-change' requires a 'changes' field. If this artifact isn't a configuration change, use type 'custom' instead.`
+          };
+
+          if (typeHints[type]) {
+            errorMsg = `Schema validation failed: ${typeHints[type]}`;
+          }
+
+          errors.push(errorMsg);
+        }
+      }
+    }
+
+    // Check types for each field in schema.properties
+    if (schema.properties) {
+      for (const [field, fieldSchema] of Object.entries(schema.properties)) {
+        // Only validate if field exists in data
+        if (field in data) {
+          const value = data[field];
+          const expectedType = fieldSchema.type;
+
+          // Type checking
+          if (expectedType) {
+            let actualType = typeof value;
+
+            // Special case: arrays
+            if (expectedType === 'array') {
+              if (!Array.isArray(value)) {
+                errors.push(`field '${field}' must be an array`);
+                continue;
+              }
+
+              // Check array items if schema defines items
+              if (fieldSchema.items) {
+                const itemsSchema = fieldSchema.items;
+
+                // Check each array item
+                for (let i = 0; i < value.length; i++) {
+                  const item = value[i];
+
+                  // Check item type
+                  if (itemsSchema.type) {
+                    const itemActualType = typeof item;
+                    if (itemsSchema.type === 'object' && itemActualType !== 'object') {
+                      errors.push(`field '${field}[${i}]' must be an object`);
+                    } else if (itemsSchema.type !== 'object' && itemActualType !== itemsSchema.type) {
+                      errors.push(`field '${field}[${i}]' must be a ${itemsSchema.type}`);
+                    }
+                  }
+
+                  // Check required fields in array items
+                  if (itemsSchema.required && typeof item === 'object') {
+                    for (const requiredField of itemsSchema.required) {
+                      if (!(requiredField in item)) {
+                        errors.push(`field '${field}[${i}].${requiredField}' is required`);
+                      }
+                    }
+                  }
+
+                  // Check enum values in array items
+                  if (itemsSchema.properties) {
+                    for (const [itemField, itemFieldSchema] of Object.entries(itemsSchema.properties)) {
+                      if (itemField in item && itemFieldSchema.enum) {
+                        if (!itemFieldSchema.enum.includes(item[itemField])) {
+                          errors.push(`field '${field}[${i}].${itemField}' must be one of: ${itemFieldSchema.enum.join(', ')}`);
+                        }
+                      }
+                    }
+                  }
+                }
+              }
+            } else if (expectedType === 'object') {
+              // Check if value is an object (but not null or array)
+              if (actualType !== 'object' || value === null || Array.isArray(value)) {
+                errors.push(`field '${field}' must be an object`);
+              }
+            } else if (expectedType === 'number') {
+              if (actualType !== 'number') {
+                errors.push(`field '${field}' must be a number`);
+              }
+            } else if (expectedType === 'string') {
+              if (actualType !== 'string') {
+                errors.push(`field '${field}' must be a string`);
+              }
+            } else if (expectedType === 'boolean') {
+              if (actualType !== 'boolean') {
+                errors.push(`field '${field}' must be a boolean`);
+              }
+            }
+          }
+
+          // Enum validation
+          if (fieldSchema.enum && !fieldSchema.enum.includes(value)) {
+            errors.push(`field '${field}' must be one of: ${fieldSchema.enum.join(', ')}`);
+          }
+        }
+      }
+    }
+
+    // Return validation result
+    if (errors.length > 0) {
+      return { valid: false, errors };
+    }
+
+    return { valid: true };
+  }
+
+  /**
+   * Create default schemas for well-known artifact types
+   * @private
+   */
+  _createDefaultSchemas() {
+    // Define default schemas for well-known types
+    const defaultSchemas = {
+      'api-contract': {
+        type: 'object',
+        required: ['endpoints'],
+        properties: {
+          endpoints: {
+            type: 'array',
+            items: {
+              type: 'object',
+              required: ['method', 'path'],
+              properties: {
+                method: {
+                  type: 'string',
+                  enum: ['GET', 'POST', 'PUT', 'PATCH', 'DELETE']
+                },
+                path: {
+                  type: 'string'
+                }
+              }
+            }
+          }
+        }
+      },
+      'schema-change': {
+        type: 'object',
+        required: ['models'],
+        properties: {
+          models: {
+            type: 'array',
+            items: {
+              type: 'object',
+              required: ['name'],
+              properties: {
+                name: {
+                  type: 'string'
+                }
+              }
+            }
+          }
+        }
+      },
+      'test-results': {
+        type: 'object',
+        required: ['total', 'passed', 'failed'],
+        properties: {
+          total: {
+            type: 'number'
+          },
+          passed: {
+            type: 'number'
+          },
+          failed: {
+            type: 'number'
+          }
+        }
+      },
+      'component-spec': {
+        type: 'object',
+        required: ['componentName'],
+        properties: {
+          componentName: {
+            type: 'string'
+          }
+        }
+      },
+      'file-manifest': {
+        type: 'object',
+        required: ['files'],
+        properties: {
+          files: {
+            type: 'array'
+          }
+        }
+      },
+      'config-change': {
+        type: 'object',
+        required: ['changes'],
+        properties: {
+          changes: {
+            type: 'array'
+          }
+        }
+      }
+    };
+
+    // Write each schema file if it doesn't exist
+    for (const [type, schema] of Object.entries(defaultSchemas)) {
+      const schemaPath = path.join(this.schemasDir, `${type}.json`);
+
+      if (!fs.existsSync(schemaPath)) {
+        try {
+          fs.writeFileSync(schemaPath, JSON.stringify(schema, null, 2), 'utf8');
+        } catch (err) {
+          // Ignore errors - schemas are optional
+        }
+      }
+    }
+  }
+
+  /**
+   * Calculate SHA-256 checksum for data
+   * @param {Object} data - The data to checksum
+   * @returns {string} Checksum prefixed with 'sha256:'
+   * @private
+   */
+  _calculateChecksum(data) {
+    const hash = crypto.createHash('sha256')
+      .update(JSON.stringify(data))
+      .digest('hex');
+    return `sha256:${hash}`;
+  }
+
+  /**
+   * Publish a new version of an artifact
+   * @param {string} artifactId - Unique identifier for the artifact
+   * @param {Object} options - Publication options
+   * @param {string} options.type - Artifact type (e.g., 'api-contract')
+   * @param {string} options.name - Human-readable name
+   * @param {string} [options.summary=''] - Brief description of this version
+   * @param {Object} options.data - The actual artifact data
+   * @param {string[]} [options.tags=[]] - Tags for categorization
+   * @param {string} options.publisher - Who/what published this version
+   * @param {string[]} [options.derivedFrom=[]] - Artifact IDs this was derived from
+   * @returns {Object} { artifactId, version, type }
+   */
+  publish(artifactId, { type, name, summary = '', data, tags = [], publisher, derivedFrom = [] }) {
+    // Step 1: Validate data against schema
+    const validation = this._validateData(type, data);
+    if (!validation.valid) {
+      throw new Error(`Schema validation failed: ${validation.errors.join(', ')}`);
+    }
+
+    // Step 2: Acquire lock on the artifacts index
+    acquireLock(this.locksDir, 'artifacts-index');
+
+    try {
+      // Step 3: Read the current index
+      const index = readJsonSafe(this.indexPath, {});
+
+      // Step 4: Determine the version number
+      let version = 1;
+      let createdAt = new Date().toISOString();
+
+      if (index[artifactId]) {
+        // Artifact exists - increment version
+        version = index[artifactId].latestVersion + 1;
+        createdAt = index[artifactId].createdAt; // Preserve original creation time
+      }
+
+      // Step 5: Create version file data
+      const versionData = {
+        artifactId,
+        version,
+        type,
+        publisher,
+        publishedAt: new Date().toISOString(),
+        summary,
+        name,
+        data,
+        tags,
+        lineage: {
+          producedBy: null,
+          derivedFrom
+        },
+        checksum: this._calculateChecksum(data)
+      };
+
+      // Step 6: Write the version file (immutable)
+      const artifactDir = path.join(this.dataDir, artifactId);
+
+      // Ensure artifact directory exists
+      if (!fs.existsSync(artifactDir)) {
+        fs.mkdirSync(artifactDir, { recursive: true });
+      }
+
+      const versionPath = path.join(artifactDir, `v${version}.json`);
+
+      try {
+        // Attempt to write the immutable version file
+        writeImmutable(versionPath, versionData);
+      } catch (err) {
+        // Handle race condition - file already exists
+        if (err.code === 'EEXIST') {
+          // Re-read index to get actual latest version
+          const freshIndex = readJsonSafe(this.indexPath, {});
+          const freshLatest = freshIndex[artifactId]?.latestVersion || version;
+          version = freshLatest + 1;
+          const updatedData = { ...versionData, version, publishedAt: new Date().toISOString() };
+          const retryPath = path.join(artifactDir, `v${version}.json`);
+          writeImmutable(retryPath, updatedData);
+          versionData = updatedData; // Update for index write below
+        } else {
+          throw err;
+        }
+      }
+
+      // Step 7: Update the index entry
+      index[artifactId] = {
+        artifactId,
+        type,
+        name,
+        publisher,
+        createdAt,
+        updatedAt: new Date().toISOString(),
+        latestVersion: version,
+        tags
+      };
+
+      // Step 8: Save the index atomically
+      atomicWriteJson(this.indexPath, index);
+
+      // Step 9: Release the lock
+      releaseLock(this.locksDir, 'artifacts-index');
+
+      // Step 10: Return publication result
+      return { artifactId, version, type };
+
+    } catch (err) {
+      // Ensure lock is released even on error
+      releaseLock(this.locksDir, 'artifacts-index');
+      throw err;
+    }
+  }
+
+  /**
+   * Get a specific version of an artifact
+   * @param {string} artifactId - The artifact identifier
+   * @param {number|null} [version=null] - Version number (null = latest)
+   * @returns {Object|null} The version data, or null if not found
+   */
+  get(artifactId, version = null) {
+    // If version is null, get the latest version from index
+    if (version === null) {
+      const index = readJsonSafe(this.indexPath, {});
+
+      if (!index[artifactId]) {
+        return null; // Artifact doesn't exist
+      }
+
+      version = index[artifactId].latestVersion;
+    }
+
+    // Read the version file
+    const versionPath = path.join(this.dataDir, artifactId, `v${version}.json`);
+
+    if (!fs.existsSync(versionPath)) {
+      return null; // Version doesn't exist
+    }
+
+    try {
+      const content = fs.readFileSync(versionPath, 'utf8');
+      return JSON.parse(content);
+    } catch (err) {
+      return null; // Corrupted or invalid file
+    }
+  }
+
+  /**
+   * Track that a session read an artifact
+   * @param {string} artifactId - The artifact that was read
+   * @param {string} reader - The session name that read it
+   * @param {number} version - The version that was read
+   */
+  trackRead(artifactId, reader, version) {
+    if (!reader) return; // Skip if no reader identified
+
+    const readsPath = path.join(this.dataDir, artifactId, 'reads.json');
+
+    // Read existing reads log
+    let reads = readJsonSafe(readsPath, []);
+
+    // Add this read event
+    reads.push({
+      reader,
+      version,
+      readAt: new Date().toISOString(),
+    });
+
+    // Write back (not immutable — reads are append-only log)
+    const dir = path.join(this.dataDir, artifactId);
+    if (!fs.existsSync(dir)) {
+      fs.mkdirSync(dir, { recursive: true });
+    }
+    fs.writeFileSync(readsPath, JSON.stringify(reads, null, 2));
+  }
+
+  /**
+   * Get the read log for an artifact
+   * @param {string} artifactId - The artifact to check
+   * @returns {Array} Array of read events: [{ reader, version, readAt }]
+   */
+  getReads(artifactId) {
+    const readsPath = path.join(this.dataDir, artifactId, 'reads.json');
+    return readJsonSafe(readsPath, []);
+  }
+
+  /**
+   * List artifacts with optional filtering
+   * @param {Object} [filters={}] - Filter criteria
+   * @param {string} [filters.type] - Filter by artifact type
+   * @param {string} [filters.publisher] - Filter by publisher
+   * @param {string} [filters.tag] - Filter by tag
+   * @returns {Array} Array of index entries
+   */
+  list({ type, publisher, tag } = {}) {
+    // Read the index
+    const index = readJsonSafe(this.indexPath, {});
+
+    // Convert index object to array
+    let results = Object.values(index);
+
+    // Apply filters
+    if (type) {
+      results = results.filter(item => item.type === type);
+    }
+
+    if (publisher) {
+      results = results.filter(item => item.publisher === publisher);
+    }
+
+    if (tag) {
+      results = results.filter(item => item.tags && item.tags.includes(tag));
+    }
+
+    // Enrich with read counts
+    results = results.map(item => {
+      const reads = this.getReads(item.artifactId);
+      const uniqueReaders = [...new Set(reads.map(r => r.reader))];
+      return {
+        ...item,
+        readCount: reads.length,
+        uniqueReaders,
+      };
+    });
+
+    return results;
+  }
+
+  /**
+   * Get version history for an artifact
+   * @param {string} artifactId - The artifact identifier
+   * @returns {Array} Array of version summaries
+   */
+  history(artifactId) {
+    // Read the index to get latest version
+    const index = readJsonSafe(this.indexPath, {});
+
+    if (!index[artifactId]) {
+      return []; // Artifact doesn't exist
+    }
+
+    const latestVersion = index[artifactId].latestVersion;
+    const artifactDir = path.join(this.dataDir, artifactId);
+    const history = [];
+
+    // Read all version files from v1 to vN
+    for (let v = 1; v <= latestVersion; v++) {
+      const versionPath = path.join(artifactDir, `v${v}.json`);
+
+      if (fs.existsSync(versionPath)) {
+        try {
+          const content = fs.readFileSync(versionPath, 'utf8');
+          const versionData = JSON.parse(content);
+
+          // Add summary to history
+          history.push({
+            version: versionData.version,
+            publishedAt: versionData.publishedAt,
+            publisher: versionData.publisher,
+            summary: versionData.summary,
+            checksum: versionData.checksum
+          });
+        } catch (err) {
+          // Skip corrupted version files
+        }
+      }
+    }
+
+    return history;
+  }
+
+  /**
+   * Repair the index by scanning all version files
+   * Useful for recovering from index corruption
+   * @returns {Object} { repaired: count }
+   */
+  repair() {
+    // Acquire lock on the artifacts index
+    acquireLock(this.locksDir, 'artifacts-index');
+
+    try {
+      const newIndex = {};
+      let repairedCount = 0;
+
+      // Scan the data directory for all artifact directories
+      if (!fs.existsSync(this.dataDir)) {
+        releaseLock(this.locksDir, 'artifacts-index');
+        return { repaired: 0 };
+      }
+
+      const artifactDirs = fs.readdirSync(this.dataDir);
+
+      // Process each artifact directory
+      for (const artifactId of artifactDirs) {
+        const artifactDir = path.join(this.dataDir, artifactId);
+
+        // Skip if not a directory
+        if (!fs.statSync(artifactDir).isDirectory()) {
+          continue;
+        }
+
+        // Find all version files
+        const files = fs.readdirSync(artifactDir);
+        const versionFiles = files.filter(f => f.match(/^v\d+\.json$/));
+
+        if (versionFiles.length === 0) {
+          continue; // No version files
+        }
+
+        // Sort version files numerically
+        versionFiles.sort((a, b) => {
+          const versionA = parseInt(a.match(/\d+/)[0]);
+          const versionB = parseInt(b.match(/\d+/)[0]);
+          return versionA - versionB;
+        });
+
+        // Read the latest version file
+        const latestVersionFile = versionFiles[versionFiles.length - 1];
+        const latestVersionPath = path.join(artifactDir, latestVersionFile);
+
+        try {
+          const content = fs.readFileSync(latestVersionPath, 'utf8');
+          const versionData = JSON.parse(content);
+
+          // Read the first version file for createdAt
+          const firstVersionPath = path.join(artifactDir, versionFiles[0]);
+          const firstContent = fs.readFileSync(firstVersionPath, 'utf8');
+          const firstVersionData = JSON.parse(firstContent);
+
+          // Rebuild index entry
+          newIndex[artifactId] = {
+            artifactId,
+            type: versionData.type,
+            name: versionData.name,
+            publisher: versionData.publisher,
+            createdAt: firstVersionData.publishedAt,
+            updatedAt: versionData.publishedAt,
+            latestVersion: versionData.version,
+            tags: versionData.tags || []
+          };
+
+          repairedCount++;
+        } catch (err) {
+          // Skip corrupted artifacts
+        }
+      }
+
+      // Write the rebuilt index atomically
+      atomicWriteJson(this.indexPath, newIndex);
+
+      // Release lock
+      releaseLock(this.locksDir, 'artifacts-index');
+
+      return { repaired: repairedCount };
+
+    } catch (err) {
+      // Ensure lock is released even on error
+      releaseLock(this.locksDir, 'artifacts-index');
+      throw err;
+    }
+  }
+}
+
+// Export the ArtifactStore class
+module.exports = ArtifactStore;