@reshotdev/screenshot 0.0.1-beta.0

package/src/commands/ingest.js

@@ -0,0 +1,458 @@
+// ingest.js - Upload traces and documentation for DocSync processing
+// Implements the "Smart Handoff" protocol from the DocSync specification
+
+const chalk = require("chalk");
+const crypto = require("crypto");
+const fs = require("fs-extra");
+const path = require("path");
+const { execSync } = require("child_process");
+const config = require("../lib/config");
+const apiClient = require("../lib/api-client");
+const { hashFile } = require("../lib/hash");
+const pkg = require("../../package.json");
+
+// File extension allowlists
+const TRACE_EXTENSIONS = [".zip"];
+const DOC_EXTENSIONS = [".md", ".mdx"];
+const MAX_DOC_SIZE = 5 * 1024 * 1024; // 5MB per doc file
+const MAX_TRACE_SIZE = 100 * 1024 * 1024; // 100MB per trace
+
+/**
+ * Parse frontmatter from a markdown file
+ * Extracts YAML frontmatter between --- delimiters
+ */
+function parseFrontmatter(content) {
+  const frontmatterRegex = /^---\n([\s\S]*?)\n---/;
+  const match = content.match(frontmatterRegex);
+  
+  if (!match) return { frontmatter: {}, content };
+  
+  const frontmatter = {};
+  const lines = match[1].split('\n');
+  
+  for (const line of lines) {
+    const colonIndex = line.indexOf(':');
+    if (colonIndex > 0) {
+      const key = line.slice(0, colonIndex).trim();
+      let value = line.slice(colonIndex + 1).trim();
+      // Remove quotes if present
+      if ((value.startsWith('"') && value.endsWith('"')) ||
+          (value.startsWith("'") && value.endsWith("'"))) {
+        value = value.slice(1, -1);
+      }
+      frontmatter[key] = value;
+    }
+  }
+  
+  return {
+    frontmatter,
+    content: content.slice(match[0].length).trim()
+  };
+}
+
+/**
+ * Discover documentation files based on docsync.config.json
+ */
+async function discoverDocumentation(docConfig, projectRoot) {
+  const files = [];
+  const root = path.resolve(projectRoot, docConfig.root || './docs');
+  
+  if (!fs.existsSync(root)) {
+    console.log(chalk.yellow(`  ⚠ Documentation root not found: ${root}`));
+    return files;
+  }
+  
+  const include = docConfig.include || ['**/*.md', '**/*.mdx'];
+  const exclude = docConfig.exclude || ['**/_*.mdx', 'node_modules'];
+  const mappings = docConfig.mappings || {};
+  
+  // Simple glob-like recursive file discovery
+  function walkDir(dir, relativePath = '') {
+    const items = fs.readdirSync(dir);
+    
+    for (const item of items) {
+      const fullPath = path.join(dir, item);
+      const relPath = path.join(relativePath, item);
+      const stat = fs.statSync(fullPath);
+      
+      // Check exclusions
+      const shouldExclude = exclude.some(pattern => {
+        if (pattern.includes('*')) {
+          // Simple glob matching
+          const regex = new RegExp(pattern.replace(/\*\*/g, '.*').replace(/\*/g, '[^/]*'));
+          return regex.test(relPath);
+        }
+        return relPath.includes(pattern) || item === pattern;
+      });
+      
+      if (shouldExclude) continue;
+      
+      if (stat.isDirectory()) {
+        walkDir(fullPath, relPath);
+      } else {
+        const ext = path.extname(item).toLowerCase();
+        if (DOC_EXTENSIONS.includes(ext)) {
+          // Read file and check for reshot_journey frontmatter
+          const content = fs.readFileSync(fullPath, 'utf-8');
+          const { frontmatter } = parseFrontmatter(content);
+          
+          // Get journey key from frontmatter or explicit mappings
+          const journeyKey = frontmatter.reshot_journey || 
+                            mappings[relPath] || 
+                            mappings[fullPath];
+          
+          // Only include files that have a binding
+          if (journeyKey) {
+            const fileSize = stat.size;
+            
+            if (fileSize > MAX_DOC_SIZE) {
+              console.log(chalk.yellow(`  ⚠ Skipping ${relPath}: exceeds ${MAX_DOC_SIZE / 1024 / 1024}MB limit`));
+              continue;
+            }
+            
+            files.push({
+              path: fullPath,
+              relativePath: relPath,
+              journeyKey,
+              contentHash: crypto.createHash('sha256').update(content).digest('hex'),
+              size: fileSize,
+              frontmatter
+            });
+          }
+        }
+      }
+    }
+  }
+  
+  walkDir(root);
+  return files;
+}
+
+/**
+ * Discover Playwright trace files
+ */
+async function discoverTraces(traceDir) {
+  const traces = [];
+  
+  if (!fs.existsSync(traceDir)) {
+    console.log(chalk.yellow(`  ⚠ Trace directory not found: ${traceDir}`));
+    return traces;
+  }
+  
+  function walkDir(dir) {
+    const items = fs.readdirSync(dir);
+    
+    for (const item of items) {
+      const fullPath = path.join(dir, item);
+      const stat = fs.statSync(fullPath);
+      
+      if (stat.isDirectory()) {
+        walkDir(fullPath);
+      } else if (path.extname(item).toLowerCase() === '.zip') {
+        const fileSize = stat.size;
+        
+        if (fileSize > MAX_TRACE_SIZE) {
+          console.log(chalk.yellow(`  ⚠ Skipping trace ${item}: exceeds ${MAX_TRACE_SIZE / 1024 / 1024}MB limit`));
+          continue;
+        }
+        
+        // Extract journey key from directory structure or filename
+        // Convention: test-results/<test-name>/trace.zip
+        const parentDir = path.basename(path.dirname(fullPath));
+        const journeyKey = parentDir !== 'test-results' ? parentDir : path.basename(item, '.zip');
+        
+        traces.push({
+          path: fullPath,
+          filename: item,
+          journeyKey,
+          contentHash: hashFile(fullPath),
+          size: fileSize
+        });
+      }
+    }
+  }
+  
+  walkDir(traceDir);
+  return traces;
+}
+
+/**
+ * Get git metadata for the current commit
+ */
+function getGitMetadata() {
+  try {
+    const commitHash = execSync('git rev-parse HEAD', { encoding: 'utf-8' }).trim();
+    const branch = execSync('git rev-parse --abbrev-ref HEAD', { encoding: 'utf-8' }).trim();
+    const commitMessage = execSync('git log -1 --pretty=%B', { encoding: 'utf-8' }).trim();
+    
+    return { commitHash, branch, commitMessage };
+  } catch {
+    console.log(chalk.yellow('  ⚠ Not a git repository or git not available'));
+    return { commitHash: 'unknown', branch: 'unknown', commitMessage: '' };
+  }
+}
+
+/**
+ * Phase 1: Initialize ingestion with manifest handshake
+ */
+async function initializeIngestion(apiKey, projectId, manifest) {
+  const response = await apiClient.post('/v1/ingest/init', {
+    projectId,
+    manifest
+  }, {
+    headers: { Authorization: `Bearer ${apiKey}` }
+  });
+  
+  return response;
+}
+
+/**
+ * Phase 2: Upload files to presigned URLs
+ */
+async function uploadFiles(files, presignedUrls, onProgress) {
+  const results = [];
+  let uploaded = 0;
+  
+  for (const file of files) {
+    const presigned = presignedUrls[file.contentHash];
+    
+    if (!presigned) {
+      // File already exists (deduplication)
+      results.push({ ...file, skipped: true });
+      uploaded++;
+      onProgress?.(uploaded, files.length);
+      continue;
+    }
+    
+    try {
+      const fileBuffer = fs.readFileSync(file.path);
+      
+      // Upload to presigned URL
+      await apiClient.uploadToPresignedUrl(presigned.url, fileBuffer, {
+        contentType: presigned.contentType || 'application/octet-stream'
+      });
+      
+      results.push({ ...file, uploaded: true, storageKey: presigned.storageKey });
+    } catch (error) {
+      results.push({ ...file, error: error.message });
+    }
+    
+    uploaded++;
+    onProgress?.(uploaded, files.length);
+  }
+  
+  return results;
+}
+
+/**
+ * Phase 3: Commit the ingestion job
+ */
+async function commitIngestion(apiKey, projectId, uploadResults, git, cliVersion) {
+  const response = await apiClient.post('/v1/ingest/commit', {
+    projectId,
+    uploadResults,
+    git,
+    cli: {
+      version: cliVersion,
+      timestamp: new Date().toISOString()
+    }
+  }, {
+    headers: { Authorization: `Bearer ${apiKey}` }
+  });
+  
+  return response;
+}
+
+/**
+ * Main ingest command
+ */
+async function ingestCommand(options = {}) {
+  console.log(chalk.blue('\n📥 Reshot DocSync Ingest\n'));
+  
+  // Read configuration
+  let docSyncConfig;
+  try {
+    docSyncConfig = config.readDocSyncConfig();
+  } catch (error) {
+    console.error(chalk.red('Error:'), 'docsync.config.json not found. Run `reshot init` first.');
+    process.exit(1);
+  }
+  
+  // Get API key and project ID
+  const settings = config.readSettings();
+  const apiKey = process.env.RESHOT_API_KEY || settings?.apiKey;
+  const projectId = process.env.RESHOT_PROJECT_ID || 
+                    settings?.projectId || 
+                    docSyncConfig._metadata?.projectId;
+  
+  if (!apiKey) {
+    console.error(chalk.red('Error:'), 'API key not found. Set RESHOT_API_KEY or run `reshot auth`.');
+    process.exit(1);
+  }
+  
+  if (!projectId) {
+    console.error(chalk.red('Error:'), 'Project ID not found. Set RESHOT_PROJECT_ID or run `reshot init`.');
+    process.exit(1);
+  }
+  
+  // Validate documentation configuration
+  const docConfig = docSyncConfig.documentation;
+  if (!docConfig) {
+    console.error(chalk.red('Error:'), 'No "documentation" block found in docsync.config.json');
+    process.exit(1);
+  }
+  
+  if (!docConfig.strategy) {
+    console.error(chalk.red('Error:'), 'documentation.strategy is required (git_pr or external_host)');
+    process.exit(1);
+  }
+  
+  const projectRoot = process.cwd();
+  const traceDir = options.traceDir || path.join(projectRoot, 'test-results');
+  
+  // Phase 1: Discovery
+  console.log(chalk.gray('📁 Discovering files...'));
+  
+  const [docs, traces] = await Promise.all([
+    discoverDocumentation(docConfig, projectRoot),
+    discoverTraces(traceDir)
+  ]);
+  
+  console.log(chalk.green(`  ✓ Found ${docs.length} documentation file(s) with journey bindings`));
+  console.log(chalk.green(`  ✓ Found ${traces.length} trace file(s)`));
+  
+  if (docs.length === 0 && traces.length === 0) {
+    console.log(chalk.yellow('\n⚠ No files to ingest. Make sure:'));
+    console.log(chalk.yellow('  - Documentation files have reshot_journey frontmatter'));
+    console.log(chalk.yellow('  - Playwright traces are in test-results/'));
+    return;
+  }
+  
+  // Validate unique bindings (1:1 relationship)
+  const journeyKeys = new Set();
+  const duplicates = [];
+  for (const doc of docs) {
+    if (journeyKeys.has(doc.journeyKey)) {
+      duplicates.push(doc.journeyKey);
+    }
+    journeyKeys.add(doc.journeyKey);
+  }
+  
+  if (duplicates.length > 0) {
+    console.error(chalk.red('\nError: Duplicate journey bindings found:'));
+    duplicates.forEach(key => console.error(chalk.red(`  - ${key}`)));
+    console.error(chalk.yellow('Each document must have a unique reshot_journey key.'));
+    process.exit(1);
+  }
+  
+  // Get git metadata
+  const git = getGitMetadata();
+  console.log(chalk.gray(`\n📍 Git: ${git.branch} @ ${git.commitHash.slice(0, 7)}`));
+  
+  // Dry run mode
+  if (options.dryRun) {
+    console.log(chalk.blue('\n🔍 Dry run - would upload:'));
+    console.log(chalk.gray('\nDocumentation:'));
+    docs.forEach(doc => {
+      console.log(chalk.white(`  ${doc.relativePath} → ${doc.journeyKey}`));
+    });
+    console.log(chalk.gray('\nTraces:'));
+    traces.forEach(trace => {
+      console.log(chalk.white(`  ${trace.filename} → ${trace.journeyKey}`));
+    });
+    return;
+  }
+  
+  // Phase 2: Manifest Handshake
+  console.log(chalk.gray('\n🤝 Initializing ingestion...'));
+  
+  const manifest = {
+    docs: docs.map(d => ({
+      relativePath: d.relativePath,
+      journeyKey: d.journeyKey,
+      contentHash: d.contentHash,
+      size: d.size
+    })),
+    traces: traces.map(t => ({
+      filename: t.filename,
+      journeyKey: t.journeyKey,
+      contentHash: t.contentHash,
+      size: t.size
+    }))
+  };
+  
+  let initResult;
+  try {
+    initResult = await initializeIngestion(apiKey, projectId, manifest);
+    console.log(chalk.green('  ✓ Server acknowledged manifest'));
+    
+    if (initResult.skippedFiles?.length > 0) {
+      console.log(chalk.gray(`  ℹ ${initResult.skippedFiles.length} file(s) unchanged (cached)`));
+    }
+  } catch (error) {
+    console.error(chalk.red('\nError during init:'), error.message);
+    process.exit(1);
+  }
+  
+  // Phase 3: Upload files to presigned URLs
+  const filesToUpload = [...docs, ...traces].filter(
+    f => !initResult.skippedFiles?.includes(f.contentHash)
+  );
+  
+  if (filesToUpload.length > 0) {
+    console.log(chalk.gray(`\n📤 Uploading ${filesToUpload.length} file(s)...`));
+    
+    const uploadResults = await uploadFiles(
+      filesToUpload,
+      initResult.presignedUrls || {},
+      (current, total) => {
+        process.stdout.write(`\r  Progress: ${current}/${total}`);
+      }
+    );
+    
+    const failed = uploadResults.filter(r => r.error);
+    if (failed.length > 0) {
+      console.log(chalk.red(`\n  ✗ ${failed.length} upload(s) failed`));
+      failed.forEach(f => console.log(chalk.red(`    - ${f.relativePath || f.filename}: ${f.error}`)));
+    }
+    
+    console.log(chalk.green(`\n  ✓ Uploaded ${filesToUpload.length - failed.length} file(s)`));
+  }
+  
+  // Phase 4: Commit and trigger processing
+  console.log(chalk.gray('\n🔒 Committing ingestion job...'));
+  
+  try {
+    const commitResult = await commitIngestion(apiKey, projectId, {
+      docs: docs.map(d => ({
+        relativePath: d.relativePath,
+        journeyKey: d.journeyKey,
+        storageKey: initResult.presignedUrls?.[d.contentHash]?.storageKey || d.contentHash
+      })),
+      traces: traces.map(t => ({
+        filename: t.filename,
+        journeyKey: t.journeyKey,
+        storageKey: initResult.presignedUrls?.[t.contentHash]?.storageKey || t.contentHash
+      }))
+    }, git, pkg.version);
+    
+    console.log(chalk.green('  ✓ Ingestion job created'));
+    console.log(chalk.blue(`\n📊 Job ID: ${commitResult.jobId}`));
+    console.log(chalk.gray('  The background worker will process traces and detect drift.'));
+    
+    if (docConfig.strategy === 'git_pr') {
+      console.log(chalk.gray('  A Pull Request will be created if drift is detected.'));
+    } else {
+      console.log(chalk.gray('  Check the Sync Kit in the dashboard for update proposals.'));
+    }
+    
+    console.log(chalk.blue('\n✨ Ingestion complete!\n'));
+    
+    return commitResult;
+  } catch (error) {
+    console.error(chalk.red('\nError during commit:'), error.message);
+    process.exit(1);
+  }
+}
+
+module.exports = ingestCommand;