@juspay/yama 1.3.0 → 1.4.0

package/dist/utils/ParallelProcessing.d.ts

@@ -0,0 +1,112 @@
+/**
+ * Parallel Processing Utilities for Batch Processing
+ * Provides concurrency control and resource management for parallel batch execution
+ */
+import { SemaphoreInterface, TokenBudgetManagerInterface } from "../types/index.js";
+/**
+ * Semaphore for controlling concurrent access to resources
+ * Limits the number of concurrent operations that can run simultaneously
+ */
+export declare class Semaphore implements SemaphoreInterface {
+    private permits;
+    private waiting;
+    constructor(permits: number);
+    /**
+     * Acquire a permit from the semaphore
+     * If no permits are available, the caller will wait until one becomes available
+     */
+    acquire(): Promise<void>;
+    /**
+     * Release a permit back to the semaphore
+     * This will allow waiting operations to proceed
+     */
+    release(): void;
+    /**
+     * Get the number of available permits
+     */
+    getAvailablePermits(): number;
+    /**
+     * Get the number of operations waiting for permits
+     */
+    getWaitingCount(): number;
+    /**
+     * Get semaphore status for debugging
+     */
+    getStatus(): {
+        available: number;
+        waiting: number;
+    };
+}
+/**
+ * Token Budget Manager for controlling AI token usage across parallel batches
+ * Ensures that the total token usage doesn't exceed the configured limits
+ */
+export declare class TokenBudgetManager implements TokenBudgetManagerInterface {
+    private totalBudget;
+    private usedTokens;
+    private batchAllocations;
+    private reservedTokens;
+    constructor(totalBudget: number);
+    /**
+     * Allocate tokens for a specific batch
+     * Returns true if allocation was successful, false if insufficient budget
+     */
+    allocateForBatch(batchIndex: number, estimatedTokens: number): boolean;
+    /**
+     * Release tokens allocated to a batch
+     * This should be called when a batch completes (successfully or with error)
+     */
+    releaseBatch(batchIndex: number): void;
+    /**
+     * Get the available token budget (not yet allocated or used)
+     */
+    getAvailableBudget(): number;
+    /**
+     * Get the total token budget
+     */
+    getTotalBudget(): number;
+    /**
+     * Get the number of tokens actually used (completed batches)
+     */
+    getUsedTokens(): number;
+    /**
+     * Get the number of tokens reserved (allocated but not yet used)
+     */
+    getReservedTokens(): number;
+    /**
+     * Get the number of active batch allocations
+     */
+    getActiveBatches(): number;
+    /**
+     * Get detailed budget status for monitoring
+     */
+    getBudgetStatus(): {
+        total: number;
+        used: number;
+        reserved: number;
+        available: number;
+        activeBatches: number;
+        utilizationPercent: number;
+    };
+    /**
+     * Reset the budget manager (for testing or reuse)
+     */
+    reset(): void;
+    /**
+     * Update the total budget (useful for dynamic adjustment)
+     */
+    updateBudget(newBudget: number): void;
+}
+/**
+ * Factory function to create a Semaphore with validation
+ */
+export declare function createSemaphore(permits: number): Semaphore;
+/**
+ * Factory function to create a TokenBudgetManager with validation
+ */
+export declare function createTokenBudgetManager(totalBudget: number): TokenBudgetManager;
+/**
+ * Utility function to calculate optimal concurrency based on available resources
+ */
+export declare function calculateOptimalConcurrency(totalBatches: number, maxConcurrent: number, averageTokensPerBatch: number, totalTokenBudget: number): number;
+//# sourceMappingURL=ParallelProcessing.d.ts.map

package/dist/utils/ParallelProcessing.js

@@ -0,0 +1,228 @@
+/**
+ * Parallel Processing Utilities for Batch Processing
+ * Provides concurrency control and resource management for parallel batch execution
+ */
+import { logger } from "./Logger.js";
+/**
+ * Semaphore for controlling concurrent access to resources
+ * Limits the number of concurrent operations that can run simultaneously
+ */
+export class Semaphore {
+    permits;
+    waiting = [];
+    constructor(permits) {
+        if (permits <= 0) {
+            throw new Error("Semaphore permits must be greater than 0");
+        }
+        this.permits = permits;
+        logger.debug(`Semaphore created with ${permits} permits`);
+    }
+    /**
+     * Acquire a permit from the semaphore
+     * If no permits are available, the caller will wait until one becomes available
+     */
+    async acquire() {
+        if (this.permits > 0) {
+            this.permits--;
+            logger.debug(`Semaphore permit acquired, ${this.permits} remaining`);
+            return;
+        }
+        logger.debug(`Semaphore permit requested, waiting in queue (${this.waiting.length} waiting)`);
+        return new Promise((resolve) => {
+            this.waiting.push(resolve);
+        });
+    }
+    /**
+     * Release a permit back to the semaphore
+     * This will allow waiting operations to proceed
+     */
+    release() {
+        this.permits++;
+        logger.debug(`Semaphore permit released, ${this.permits} available`);
+        if (this.waiting.length > 0) {
+            const resolve = this.waiting.shift();
+            this.permits--;
+            logger.debug(`Semaphore permit granted to waiting operation, ${this.permits} remaining`);
+            resolve();
+        }
+    }
+    /**
+     * Get the number of available permits
+     */
+    getAvailablePermits() {
+        return this.permits;
+    }
+    /**
+     * Get the number of operations waiting for permits
+     */
+    getWaitingCount() {
+        return this.waiting.length;
+    }
+    /**
+     * Get semaphore status for debugging
+     */
+    getStatus() {
+        return {
+            available: this.permits,
+            waiting: this.waiting.length,
+        };
+    }
+}
+/**
+ * Token Budget Manager for controlling AI token usage across parallel batches
+ * Ensures that the total token usage doesn't exceed the configured limits
+ */
+export class TokenBudgetManager {
+    totalBudget;
+    usedTokens = 0;
+    batchAllocations = new Map();
+    reservedTokens = 0; // Tokens allocated but not yet used
+    constructor(totalBudget) {
+        if (totalBudget <= 0) {
+            throw new Error("Token budget must be greater than 0");
+        }
+        this.totalBudget = totalBudget;
+        logger.debug(`TokenBudgetManager created with budget of ${totalBudget} tokens`);
+    }
+    /**
+     * Allocate tokens for a specific batch
+     * Returns true if allocation was successful, false if insufficient budget
+     */
+    allocateForBatch(batchIndex, estimatedTokens) {
+        if (estimatedTokens <= 0) {
+            logger.warn(`Invalid token estimate for batch ${batchIndex}: ${estimatedTokens}`);
+            return false;
+        }
+        // Check if we already have an allocation for this batch
+        if (this.batchAllocations.has(batchIndex)) {
+            logger.warn(`Batch ${batchIndex} already has token allocation`);
+            return false;
+        }
+        // Check if we have enough budget
+        const totalAllocated = this.usedTokens + this.reservedTokens + estimatedTokens;
+        if (totalAllocated > this.totalBudget) {
+            logger.debug(`Insufficient token budget for batch ${batchIndex}: ` +
+                `need ${estimatedTokens}, available ${this.getAvailableBudget()}`);
+            return false;
+        }
+        // Allocate the tokens
+        this.reservedTokens += estimatedTokens;
+        this.batchAllocations.set(batchIndex, estimatedTokens);
+        logger.debug(`Allocated ${estimatedTokens} tokens for batch ${batchIndex} ` +
+            `(${this.getAvailableBudget()} remaining)`);
+        return true;
+    }
+    /**
+     * Release tokens allocated to a batch
+     * This should be called when a batch completes (successfully or with error)
+     */
+    releaseBatch(batchIndex) {
+        const allocated = this.batchAllocations.get(batchIndex);
+        if (!allocated) {
+            logger.warn(`No token allocation found for batch ${batchIndex}`);
+            return;
+        }
+        // Move from reserved to used (assuming the tokens were actually used)
+        this.reservedTokens -= allocated;
+        this.usedTokens += allocated;
+        this.batchAllocations.delete(batchIndex);
+        logger.debug(`Released ${allocated} tokens from batch ${batchIndex} ` +
+            `(${this.getAvailableBudget()} now available)`);
+    }
+    /**
+     * Get the available token budget (not yet allocated or used)
+     */
+    getAvailableBudget() {
+        return this.totalBudget - this.usedTokens - this.reservedTokens;
+    }
+    /**
+     * Get the total token budget
+     */
+    getTotalBudget() {
+        return this.totalBudget;
+    }
+    /**
+     * Get the number of tokens actually used (completed batches)
+     */
+    getUsedTokens() {
+        return this.usedTokens;
+    }
+    /**
+     * Get the number of tokens reserved (allocated but not yet used)
+     */
+    getReservedTokens() {
+        return this.reservedTokens;
+    }
+    /**
+     * Get the number of active batch allocations
+     */
+    getActiveBatches() {
+        return this.batchAllocations.size;
+    }
+    /**
+     * Get detailed budget status for monitoring
+     */
+    getBudgetStatus() {
+        const utilizationPercent = ((this.usedTokens + this.reservedTokens) / this.totalBudget) * 100;
+        return {
+            total: this.totalBudget,
+            used: this.usedTokens,
+            reserved: this.reservedTokens,
+            available: this.getAvailableBudget(),
+            activeBatches: this.batchAllocations.size,
+            utilizationPercent: Math.round(utilizationPercent * 100) / 100,
+        };
+    }
+    /**
+     * Reset the budget manager (for testing or reuse)
+     */
+    reset() {
+        this.usedTokens = 0;
+        this.reservedTokens = 0;
+        this.batchAllocations.clear();
+        logger.debug("TokenBudgetManager reset");
+    }
+    /**
+     * Update the total budget (useful for dynamic adjustment)
+     */
+    updateBudget(newBudget) {
+        if (newBudget <= 0) {
+            throw new Error("Token budget must be greater than 0");
+        }
+        const oldBudget = this.totalBudget;
+        this.totalBudget = newBudget;
+        logger.debug(`Token budget updated from ${oldBudget} to ${newBudget}`);
+        // Log warning if new budget is less than current usage
+        if (newBudget < this.usedTokens + this.reservedTokens) {
+            logger.warn(`New budget (${newBudget}) is less than current usage (${this.usedTokens + this.reservedTokens})`);
+        }
+    }
+}
+/**
+ * Factory function to create a Semaphore with validation
+ */
+export function createSemaphore(permits) {
+    return new Semaphore(permits);
+}
+/**
+ * Factory function to create a TokenBudgetManager with validation
+ */
+export function createTokenBudgetManager(totalBudget) {
+    return new TokenBudgetManager(totalBudget);
+}
+/**
+ * Utility function to calculate optimal concurrency based on available resources
+ */
+export function calculateOptimalConcurrency(totalBatches, maxConcurrent, averageTokensPerBatch, totalTokenBudget) {
+    // Don't exceed the configured maximum
+    let optimal = Math.min(maxConcurrent, totalBatches);
+    // Don't exceed what the token budget can support
+    const tokenBasedLimit = Math.floor(totalTokenBudget / averageTokensPerBatch);
+    optimal = Math.min(optimal, tokenBasedLimit);
+    // Ensure at least 1
+    optimal = Math.max(1, optimal);
+    logger.debug(`Calculated optimal concurrency: ${optimal} ` +
+        `(max: ${maxConcurrent}, batches: ${totalBatches}, token-limited: ${tokenBasedLimit})`);
+    return optimal;
+}
+//# sourceMappingURL=ParallelProcessing.js.map

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@juspay/yama",
-  "version": "1.3.0",
+  "version": "1.4.0",
   "description": "Enterprise-grade Pull Request automation toolkit with AI-powered code review and description enhancement",
   "keywords": [
     "pr",
@@ -83,7 +83,7 @@
     "check:all": "npm run lint && npm run format --check && npm run validate && npm run validate:commit"
   },
   "dependencies": {
-    "@juspay/neurolink": "^5.1.0",
+    "@juspay/neurolink": "^7.40.1",
     "@nexus2520/bitbucket-mcp-server": "^0.10.0",
     "chalk": "^4.1.2",
     "commander": "^11.0.0",
@@ -97,36 +97,36 @@
     "yaml": "^2.3.0"
   },
   "devDependencies": {
+    "@changesets/changelog-github": "^0.5.1",
+    "@changesets/cli": "^2.26.2",
+    "@eslint/js": "^9.0.0",
+    "@semantic-release/changelog": "^6.0.3",
+    "@semantic-release/commit-analyzer": "^13.0.0",
+    "@semantic-release/git": "^10.0.1",
+    "@semantic-release/github": "^11.0.0",
+    "@semantic-release/npm": "^12.0.1",
+    "@semantic-release/release-notes-generator": "^14.0.1",
     "@types/commander": "^2.12.5",
     "@types/inquirer": "^9.0.8",
     "@types/jest": "^29.0.0",
     "@types/lodash": "^4.14.0",
     "@types/node": "^20.0.0",
-    "@eslint/js": "^9.0.0",
     "@typescript-eslint/eslint-plugin": "^8.0.0",
     "@typescript-eslint/parser": "^8.0.0",
     "eslint": "^9.0.0",
+    "husky": "^9.0.0",
     "jest": "^29.0.0",
+    "lint-staged": "^15.0.0",
+    "prettier": "^3.0.0",
+    "publint": "^0.3.0",
     "rimraf": "^5.0.0",
+    "semantic-release": "^24.0.0",
     "ts-jest": "^29.0.0",
     "ts-node": "^10.0.0",
     "ts-node-dev": "^2.0.0",
     "tsc-alias": "^1.8.0",
     "typedoc": "^0.25.0",
-    "typescript": "^5.0.0",
-    "@changesets/changelog-github": "^0.5.1",
-    "@changesets/cli": "^2.26.2",
-    "@semantic-release/changelog": "^6.0.3",
-    "@semantic-release/commit-analyzer": "^13.0.0",
-    "@semantic-release/git": "^10.0.1",
-    "@semantic-release/github": "^11.0.0",
-    "@semantic-release/npm": "^12.0.1",
-    "@semantic-release/release-notes-generator": "^14.0.1",
-    "semantic-release": "^24.0.0",
-    "prettier": "^3.0.0",
-    "publint": "^0.3.0",
-    "husky": "^9.0.0",
-    "lint-staged": "^15.0.0"
+    "typescript": "^5.0.0"
   },
   "peerDependencies": {
     "typescript": ">=4.5.0"

package/yama.config.example.yaml

@@ -51,16 +51,54 @@ features:
       - "Performance bottlenecks"
       - "Error handling"
       - "Code quality"
-    
+
     # NEW: Batch Processing Configuration
     batchProcessing:
       enabled: true # Enable batch processing for large PRs
       maxFilesPerBatch: 3 # Maximum files to process in each batch
       prioritizeSecurityFiles: true # Process security-sensitive files first
-      parallelBatches: false # Process batches sequentially for reliability
+      parallelBatches: false # Keep for backward compatibility
       batchDelayMs: 1000 # Delay between batches in milliseconds
       singleRequestThreshold: 5 # Use single request for PRs with ≤5 files
 
+      # NEW: Parallel Processing Configuration
+      parallel:
+        enabled: true # Enable parallel processing by default
+        maxConcurrentBatches: 3 # Maximum concurrent batches
+        rateLimitStrategy: "fixed" # Options: fixed, adaptive
+        tokenBudgetDistribution: "equal" # Options: equal, weighted
+        failureHandling: "continue" # Options: continue, stop-all
+
+    # Multi-Instance Processing Configuration
+    multiInstance:
+      enabled: true # Enable multi-instance review
+      instanceCount: 2 # Number of instances to run in parallel
+      instances:
+        - name: "primary"
+          provider: "vertex"
+          model: "gemini-2.5-pro"
+          temperature: 0.3
+          weight: 1.0
+        - name: "secondary"
+          provider: "vertex"
+          model: "gemini-2.5-pro"
+          temperature: 0.1
+          weight: 1.0
+      deduplication:
+        enabled: true
+        similarityThreshold: 40 # Similarity percentage threshold (0-100)
+        maxCommentsToPost: 30
+        prioritizeBy: "severity"
+
+    # NEW: Semantic Comment Deduplication Configuration
+    semanticDeduplication:
+      enabled: true # Enable AI-powered semantic similarity analysis
+      similarityThreshold: 70 # Similarity percentage threshold (0-100)
+      batchSize: 15 # Number of violations to process per batch
+      timeout: "5m" # Timeout for similarity analysis
+      fallbackOnError: true # Return all violations if AI analysis fails
+      logMatches: true # Log detailed similarity match information
+
   # Description Enhancement Configuration
   descriptionEnhancement:
     enabled: true
@@ -154,8 +192,8 @@ monitoring:
 # Memory Bank Configuration
 memoryBank:
   enabled: true
-  path: "memory-bank"  # Primary path to look for memory bank files
-  fallbackPaths:       # Optional fallback paths if primary doesn't exist
+  path: "memory-bank" # Primary path to look for memory bank files
+  fallbackPaths: # Optional fallback paths if primary doesn't exist
     - "docs/memory-bank"
     - ".memory-bank"
     - "project-docs/context"