claude-autopm 1.27.0 → 1.29.0

package/lib/batch-processor-integration.js

@@ -0,0 +1,366 @@
+/**
+ * Batch Processor Integration Examples
+ *
+ * This file demonstrates how to integrate BatchProcessor with existing
+ * GitHub sync operations from pm-sync-upload-local.js
+ *
+ * @example
+ * const { batchSyncPRDs, batchSyncAll } = require('./lib/batch-processor-integration');
+ *
+ * // Batch sync all PRDs
+ * const results = await batchSyncPRDs({
+ *   basePath: '.claude',
+ *   owner: 'user',
+ *   repo: 'repository',
+ *   octokit,
+ *   dryRun: false
+ * });
+ */
+
+const fs = require('fs').promises;
+const path = require('path');
+const BatchProcessor = require('./batch-processor');
+const {
+  syncPRDToGitHub,
+  syncEpicToGitHub,
+  syncTaskToGitHub,
+  loadSyncMap,
+  saveSyncMap
+} = require('../autopm/.claude/scripts/pm-sync-upload-local');
+
+/**
+ * Batch sync PRDs to GitHub
+ *
+ * @param {Object} options
+ * @param {string} options.basePath - Base path containing PRDs
+ * @param {string} options.owner - GitHub repo owner
+ * @param {string} options.repo - GitHub repo name
+ * @param {Object} options.octokit - Octokit instance
+ * @param {boolean} options.dryRun - Dry run mode
+ * @param {number} options.maxConcurrent - Max concurrent uploads (default: 10)
+ * @param {Function} options.onProgress - Progress callback
+ * @returns {Promise<Object>} Results summary
+ */
+async function batchSyncPRDs({
+  basePath,
+  owner,
+  repo,
+  octokit,
+  dryRun = false,
+  maxConcurrent = 10,
+  onProgress = null
+}) {
+  const syncMapPath = path.join(basePath, 'sync-map.json');
+  const syncMap = await loadSyncMap(syncMapPath);
+
+  // Load all PRD files
+  const prdDir = path.join(basePath, 'prds');
+  let prdFiles = [];
+  try {
+    const files = await fs.readdir(prdDir);
+    prdFiles = files
+      .filter(f => f.endsWith('.md'))
+      .map(f => ({
+        path: path.join(prdDir, f),
+        id: f.replace('.md', ''),
+        type: 'prd'
+      }));
+  } catch (err) {
+    console.error(`Error reading PRD directory: ${err.message}`);
+    return {
+      total: 0,
+      succeeded: 0,
+      failed: 0,
+      duration: 0,
+      errors: []
+    };
+  }
+
+  // Create batch processor
+  const processor = new BatchProcessor({ maxConcurrent });
+
+  // Create wrapper function for syncPRDToGitHub
+  const syncFn = async (item, repo, octokit, syncMap, dryRun) => {
+    return await syncPRDToGitHub(item.path, repo, octokit, syncMap, dryRun);
+  };
+
+  // Batch upload
+  const results = await processor.batchUpload({
+    items: prdFiles,
+    syncFn,
+    repo: { owner, repo },
+    octokit,
+    syncMap,
+    dryRun,
+    onProgress
+  });
+
+  // Save sync map if not dry run
+  if (!dryRun && results.succeeded > 0) {
+    await saveSyncMap(syncMapPath, syncMap);
+  }
+
+  return results;
+}
+
+/**
+ * Batch sync Epics to GitHub
+ *
+ * @param {Object} options - Same as batchSyncPRDs
+ * @returns {Promise<Object>} Results summary
+ */
+async function batchSyncEpics({
+  basePath,
+  owner,
+  repo,
+  octokit,
+  dryRun = false,
+  maxConcurrent = 10,
+  onProgress = null
+}) {
+  const syncMapPath = path.join(basePath, 'sync-map.json');
+  const syncMap = await loadSyncMap(syncMapPath);
+
+  // Load all Epic files
+  const epicDir = path.join(basePath, 'epics');
+  let epicFiles = [];
+  try {
+    const dirs = await fs.readdir(epicDir);
+    for (const dir of dirs) {
+      const epicPath = path.join(epicDir, dir, 'epic.md');
+      try {
+        await fs.access(epicPath);
+        epicFiles.push({
+          path: epicPath,
+          id: dir,
+          type: 'epic'
+        });
+      } catch (err) {
+        // Skip if epic.md doesn't exist
+      }
+    }
+  } catch (err) {
+    console.error(`Error reading Epic directory: ${err.message}`);
+    return {
+      total: 0,
+      succeeded: 0,
+      failed: 0,
+      duration: 0,
+      errors: []
+    };
+  }
+
+  // Create batch processor
+  const processor = new BatchProcessor({ maxConcurrent });
+
+  // Create wrapper function for syncEpicToGitHub
+  const syncFn = async (item, repo, octokit, syncMap, dryRun) => {
+    return await syncEpicToGitHub(item.path, repo, octokit, syncMap, dryRun);
+  };
+
+  // Batch upload
+  const results = await processor.batchUpload({
+    items: epicFiles,
+    syncFn,
+    repo: { owner, repo },
+    octokit,
+    syncMap,
+    dryRun,
+    onProgress
+  });
+
+  // Save sync map if not dry run
+  if (!dryRun && results.succeeded > 0) {
+    await saveSyncMap(syncMapPath, syncMap);
+  }
+
+  return results;
+}
+
+/**
+ * Batch sync Tasks to GitHub
+ *
+ * @param {Object} options - Same as batchSyncPRDs
+ * @returns {Promise<Object>} Results summary
+ */
+async function batchSyncTasks({
+  basePath,
+  owner,
+  repo,
+  octokit,
+  dryRun = false,
+  maxConcurrent = 10,
+  onProgress = null
+}) {
+  const syncMapPath = path.join(basePath, 'sync-map.json');
+  const syncMap = await loadSyncMap(syncMapPath);
+
+  // Load all Task files
+  const epicDir = path.join(basePath, 'epics');
+  let taskFiles = [];
+  try {
+    const dirs = await fs.readdir(epicDir);
+    for (const dir of dirs) {
+      const tasksDir = path.join(epicDir, dir);
+      try {
+        const files = await fs.readdir(tasksDir);
+        for (const file of files) {
+          if (file.startsWith('task-') && file.endsWith('.md')) {
+            taskFiles.push({
+              path: path.join(tasksDir, file),
+              id: `${dir}-${file.replace('.md', '')}`,
+              type: 'task'
+            });
+          }
+        }
+      } catch (err) {
+        // Skip if directory can't be read
+      }
+    }
+  } catch (err) {
+    console.error(`Error reading Task directories: ${err.message}`);
+    return {
+      total: 0,
+      succeeded: 0,
+      failed: 0,
+      duration: 0,
+      errors: []
+    };
+  }
+
+  // Create batch processor
+  const processor = new BatchProcessor({ maxConcurrent });
+
+  // Create wrapper function for syncTaskToGitHub
+  const syncFn = async (item, repo, octokit, syncMap, dryRun) => {
+    return await syncTaskToGitHub(item.path, repo, octokit, syncMap, dryRun);
+  };
+
+  // Batch upload
+  const results = await processor.batchUpload({
+    items: taskFiles,
+    syncFn,
+    repo: { owner, repo },
+    octokit,
+    syncMap,
+    dryRun,
+    onProgress
+  });
+
+  // Save sync map if not dry run
+  if (!dryRun && results.succeeded > 0) {
+    await saveSyncMap(syncMapPath, syncMap);
+  }
+
+  return results;
+}
+
+/**
+ * Batch sync all items (PRDs, Epics, Tasks) to GitHub
+ *
+ * @param {Object} options - Sync options
+ * @returns {Promise<Object>} Combined results summary
+ */
+async function batchSyncAll({
+  basePath,
+  owner,
+  repo,
+  octokit,
+  dryRun = false,
+  maxConcurrent = 10,
+  onProgress = null
+}) {
+  const startTime = Date.now();
+  const combinedResults = {
+    prds: null,
+    epics: null,
+    tasks: null,
+    total: 0,
+    succeeded: 0,
+    failed: 0,
+    duration: 0,
+    errors: []
+  };
+
+  console.log('🚀 Starting batch sync to GitHub...\n');
+
+  // Sync PRDs
+  console.log('📋 Syncing PRDs...');
+  const prdResults = await batchSyncPRDs({
+    basePath,
+    owner,
+    repo,
+    octokit,
+    dryRun,
+    maxConcurrent,
+    onProgress: onProgress
+      ? (current, total, item) => onProgress('PRD', current, total, item)
+      : null
+  });
+  combinedResults.prds = prdResults;
+  console.log(`✅ PRDs: ${prdResults.succeeded}/${prdResults.total} succeeded in ${prdResults.duration}ms\n`);
+
+  // Sync Epics
+  console.log('📦 Syncing Epics...');
+  const epicResults = await batchSyncEpics({
+    basePath,
+    owner,
+    repo,
+    octokit,
+    dryRun,
+    maxConcurrent,
+    onProgress: onProgress
+      ? (current, total, item) => onProgress('Epic', current, total, item)
+      : null
+  });
+  combinedResults.epics = epicResults;
+  console.log(`✅ Epics: ${epicResults.succeeded}/${epicResults.total} succeeded in ${epicResults.duration}ms\n`);
+
+  // Sync Tasks
+  console.log('✓ Syncing Tasks...');
+  const taskResults = await batchSyncTasks({
+    basePath,
+    owner,
+    repo,
+    octokit,
+    dryRun,
+    maxConcurrent,
+    onProgress: onProgress
+      ? (current, total, item) => onProgress('Task', current, total, item)
+      : null
+  });
+  combinedResults.tasks = taskResults;
+  console.log(`✅ Tasks: ${taskResults.succeeded}/${taskResults.total} succeeded in ${taskResults.duration}ms\n`);
+
+  // Combine results
+  combinedResults.total = prdResults.total + epicResults.total + taskResults.total;
+  combinedResults.succeeded = prdResults.succeeded + epicResults.succeeded + taskResults.succeeded;
+  combinedResults.failed = prdResults.failed + epicResults.failed + taskResults.failed;
+  combinedResults.duration = Date.now() - startTime;
+  combinedResults.errors = [
+    ...prdResults.errors,
+    ...epicResults.errors,
+    ...taskResults.errors
+  ];
+
+  console.log('🎉 Batch sync complete!');
+  console.log(`   Total: ${combinedResults.succeeded}/${combinedResults.total} items synced`);
+  console.log(`   Duration: ${(combinedResults.duration / 1000).toFixed(2)}s`);
+
+  if (combinedResults.failed > 0) {
+    console.log(`   ⚠️  Failures: ${combinedResults.failed}`);
+    console.log('\nErrors:');
+    combinedResults.errors.forEach(err => {
+      console.log(`   - ${err.item.path}: ${err.error}`);
+    });
+  }
+
+  return combinedResults;
+}
+
+module.exports = {
+  batchSyncPRDs,
+  batchSyncEpics,
+  batchSyncTasks,
+  batchSyncAll
+};

package/lib/batch-processor.js

@@ -0,0 +1,278 @@
+/**
+ * Batch Processor for GitHub Sync Operations
+ *
+ * Handles batch uploading of PRDs, Epics, and Tasks to GitHub with:
+ * - Parallel processing with concurrency control
+ * - Rate limiting and exponential backoff
+ * - Progress tracking
+ * - Error recovery
+ * - Dry run mode
+ *
+ * @example
+ * const BatchProcessor = require('./lib/batch-processor');
+ *
+ * const processor = new BatchProcessor({
+ *   maxConcurrent: 10,
+ *   rateLimit: {
+ *     requestsPerHour: 5000,
+ *     retryDelay: 1000,
+ *     maxRetries: 3
+ *   }
+ * });
+ *
+ * const results = await processor.batchUpload({
+ *   items: ['.claude/prds/*.md'],
+ *   syncFn: syncPRDToGitHub,
+ *   repo: { owner, repo },
+ *   octokit,
+ *   syncMap,
+ *   dryRun: false,
+ *   onProgress: (current, total, item) => {
+ *     console.log(`[${current}/${total}] ${item.path}`);
+ *   }
+ * });
+ */
+
+class BatchProcessor {
+  /**
+   * Create a new BatchProcessor instance
+   *
+   * @param {Object} options - Configuration options
+   * @param {number} options.maxConcurrent - Maximum concurrent uploads (default: 10)
+   * @param {Object} options.rateLimit - Rate limiting configuration
+   * @param {number} options.rateLimit.requestsPerHour - GitHub API rate limit (default: 5000)
+   * @param {number} options.rateLimit.retryDelay - Initial retry delay in ms (default: 1000)
+   * @param {number} options.rateLimit.maxRetries - Maximum retry attempts (default: 3)
+   * @param {number} options.rateLimit.threshold - Remaining requests threshold to trigger wait (default: 10)
+   */
+  constructor(options = {}) {
+    // Validate configuration
+    if (options.maxConcurrent !== undefined) {
+      if (typeof options.maxConcurrent !== 'number' || options.maxConcurrent <= 0) {
+        throw new Error('maxConcurrent must be a positive number');
+      }
+    }
+
+    this.maxConcurrent = options.maxConcurrent || 10;
+    this.rateLimit = {
+      requestsPerHour: options.rateLimit?.requestsPerHour || 5000,
+      retryDelay: options.rateLimit?.retryDelay || 1000,
+      maxRetries: options.rateLimit?.maxRetries || 3,
+      threshold: options.rateLimit?.threshold || 10,
+      remaining: options.rateLimit?.requestsPerHour || 5000,
+      resetTime: Math.floor(Date.now() / 1000) + 3600
+    };
+  }
+
+  /**
+   * Batch upload items to GitHub
+   *
+   * @param {Object} params - Upload parameters
+   * @param {Array} params.items - Items to upload
+   * @param {Function} params.syncFn - Sync function to call for each item
+   * @param {Object} params.repo - Repository info {owner, repo}
+   * @param {Object} params.octokit - Octokit instance
+   * @param {Object} params.syncMap - Sync mapping object
+   * @param {boolean} params.dryRun - Dry run mode
+   * @param {Function} params.onProgress - Progress callback (current, total, item)
+   * @returns {Promise<Object>} Results summary
+   */
+  async batchUpload({
+    items,
+    syncFn,
+    repo,
+    octokit,
+    syncMap,
+    dryRun = false,
+    onProgress = null
+  }) {
+    const startTime = Date.now();
+    const results = {
+      total: items.length,
+      succeeded: 0,
+      failed: 0,
+      duration: 0,
+      errors: [],
+      rateLimit: {
+        remaining: this.rateLimit.remaining,
+        reset: this.rateLimit.resetTime
+      }
+    };
+
+    // Handle empty item list
+    if (items.length === 0) {
+      results.duration = Date.now() - startTime;
+      return results;
+    }
+
+    // Process items with concurrency control
+    const processingQueue = [];
+    let currentIndex = 0;
+    let completedCount = 0;
+
+    const processNextItem = async () => {
+      if (currentIndex >= items.length) {
+        return;
+      }
+
+      const item = items[currentIndex];
+      currentIndex++;
+
+      try {
+        // Check rate limit before processing
+        if (this.shouldWaitForRateLimit()) {
+          await this.waitForRateLimit();
+        }
+
+        // Execute sync with retry logic
+        await this.executeWithRetry(
+          syncFn,
+          item,
+          repo,
+          octokit,
+          syncMap,
+          dryRun
+        );
+
+        results.succeeded++;
+      } catch (error) {
+        results.failed++;
+        results.errors.push({
+          item,
+          error: error.message || String(error)
+        });
+      } finally {
+        completedCount++;
+
+        // Update progress
+        if (onProgress) {
+          onProgress(completedCount, items.length, item);
+        }
+
+        // Process next item
+        await processNextItem();
+      }
+    };
+
+    // Start initial batch of workers
+    const workerCount = Math.min(this.maxConcurrent, items.length);
+    for (let i = 0; i < workerCount; i++) {
+      processingQueue.push(processNextItem());
+    }
+
+    // Wait for all workers to complete
+    await Promise.all(processingQueue);
+
+    // Calculate final duration
+    results.duration = Date.now() - startTime;
+
+    // Update final rate limit info
+    results.rateLimit = {
+      remaining: this.rateLimit.remaining,
+      reset: this.rateLimit.resetTime
+    };
+
+    return results;
+  }
+
+  /**
+   * Execute sync function with retry logic for rate limit errors
+   *
+   * @param {Function} syncFn - Sync function to execute
+   * @param {Object} item - Item to sync
+   * @param {Object} repo - Repository info
+   * @param {Object} octokit - Octokit instance
+   * @param {Object} syncMap - Sync mapping
+   * @param {boolean} dryRun - Dry run mode
+   * @returns {Promise<Object>} Sync result
+   */
+  async executeWithRetry(syncFn, item, repo, octokit, syncMap, dryRun) {
+    let lastError = null;
+
+    for (let attempt = 1; attempt <= this.rateLimit.maxRetries; attempt++) {
+      try {
+        const result = await syncFn(item, repo, octokit, syncMap, dryRun);
+        return result;
+      } catch (error) {
+        lastError = error;
+
+        // Check if it's a rate limit error
+        if (error.status === 429) {
+          // Update rate limit from error response if available
+          if (error.response?.headers) {
+            this.updateRateLimit(error.response.headers);
+          }
+
+          // Calculate backoff delay for this attempt
+          const backoffDelay = this.calculateBackoffDelay(attempt);
+
+          // Wait before retrying
+          await new Promise(resolve => setTimeout(resolve, backoffDelay));
+
+          // Continue to next retry attempt
+          continue;
+        }
+
+        // For non-rate-limit errors, throw immediately
+        throw error;
+      }
+    }
+
+    // If we exhausted all retries, throw the last error
+    throw lastError;
+  }
+
+  /**
+   * Calculate exponential backoff delay
+   *
+   * @param {number} attempt - Current attempt number (1-based)
+   * @returns {number} Delay in milliseconds
+   */
+  calculateBackoffDelay(attempt) {
+    // Exponential backoff: delay * (2 ^ (attempt - 1))
+    return this.rateLimit.retryDelay * Math.pow(2, attempt - 1);
+  }
+
+  /**
+   * Update rate limit information from response headers
+   *
+   * @param {Object} headers - Response headers
+   */
+  updateRateLimit(headers) {
+    if (headers['x-ratelimit-remaining']) {
+      this.rateLimit.remaining = parseInt(headers['x-ratelimit-remaining'], 10);
+    }
+
+    if (headers['x-ratelimit-reset']) {
+      this.rateLimit.resetTime = parseInt(headers['x-ratelimit-reset'], 10);
+    }
+  }
+
+  /**
+   * Check if we should wait for rate limit to reset
+   *
+   * @returns {boolean} True if should wait
+   */
+  shouldWaitForRateLimit() {
+    return this.rateLimit.remaining <= this.rateLimit.threshold;
+  }
+
+  /**
+   * Wait for rate limit to reset
+   *
+   * @returns {Promise<void>}
+   */
+  async waitForRateLimit() {
+    const now = Math.floor(Date.now() / 1000);
+    const waitSeconds = Math.max(0, this.rateLimit.resetTime - now);
+
+    if (waitSeconds > 0) {
+      await new Promise(resolve => setTimeout(resolve, waitSeconds * 1000));
+    }
+
+    // Reset to full limit after waiting
+    this.rateLimit.remaining = this.rateLimit.requestsPerHour;
+  }
+}
+
+module.exports = BatchProcessor;