@freshworks/shiftleft-tools 1.1.8

package/templates/postman/scripts/runners/run-tests-staging.sh

@@ -0,0 +1,741 @@
+#!/usr/bin/env bash
+
+###############################################################################
+# Staging Test Runner
+#
+# Runs API tests against staging environment with automatic JWT generation.
+#
+# Usage:
+#   ./run-tests-staging.sh                    # Run V3 + V2 tests (Legacy skipped)
+#   ./run-tests-staging.sh --v3               # Run only V3 tests
+#   ./run-tests-staging.sh --v2               # Run only V2 tests
+#   ./run-tests-staging.sh --legacy           # Run only Legacy tests
+#   ./run-tests-staging.sh --multi-product    # Run V3 tests across all products
+#   ./run-tests-staging.sh --help             # Show this help
+#
+# Note: Legacy API tests are SKIPPED by default in staging.
+# Multi-Product Testing:
+#   Tests V3 APIs (CRUD + Files) across Freshdesk, Freshservice, and Freshworks CRM
+#   Requires product-specific environment files in postman/environments/:
+#     - staging-freshdesk.postman_environment.json
+#     - staging-freshservice.postman_environment.json
+#     - staging-freshworks_crm.postman_environment.json
+#
+# Prerequisites:
+#   - AWS CLI configured (set AWS_PROFILE for local use)
+#   - jq installed
+#   - Node.js installed
+#
+# Examples:
+#   AWS_PROFILE=staging ./run-tests-staging.sh
+#   ./run-tests-staging.sh --v3                  # Quick V3 test only
+#   ./run-tests-staging.sh --multi-product       # Test all products
+###############################################################################
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+PROJECT_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)"
+# Display name shown on the reports — the repo/service name, overridable.
+REPORT_LOGO="${SHIFTLEFT_REPORT_LOGO:-$(basename "$PROJECT_ROOT")}"
+POSTMAN_DIR="$PROJECT_ROOT/postman"
+
+# Configuration
+CONFIG_FILE="$POSTMAN_DIR/config/staging.json"
+REPORT_DIR="$POSTMAN_DIR/reports"
+TIMESTAMP=$(date +%Y%m%d-%H%M%S)
+
+# Collections
+# Collections (now split into multiple files by folder)
+CRUD_COLLECTIONS_DIR="$POSTMAN_DIR/collections/crud"
+FILES_COLLECTIONS_DIR="$POSTMAN_DIR/collections/files"
+V2_COLLECTIONS_DIR="$POSTMAN_DIR/collections/v2"
+LEGACY_COLLECTIONS_DIR="$POSTMAN_DIR/collections/legacy"
+ENVIRONMENT="$POSTMAN_DIR/environments/staging.postman_environment.json"
+
+# Parse arguments
+RUN_V3=false
+RUN_V2=false
+RUN_LEGACY=false
+SHOW_HELP=false
+MULTI_PRODUCT=false
+RUN_ALL=true
+
+# Multi-product configuration
+PRODUCTS="freshdesk freshservice freshworks-crm"
+declare -A PRODUCT_DISPLAY_NAMES
+PRODUCT_DISPLAY_NAMES["freshdesk"]="Freshdesk"
+PRODUCT_DISPLAY_NAMES["freshservice"]="Freshservice"
+PRODUCT_DISPLAY_NAMES["freshworks-crm"]="Freshworks CRM"
+declare -A PRODUCT_IDS
+PRODUCT_IDS["freshdesk"]="9"
+PRODUCT_IDS["freshservice"]="3"
+PRODUCT_IDS["freshworks-crm"]="39"
+declare -A PRODUCT_NAMES
+PRODUCT_NAMES["freshdesk"]="freshdesk"
+PRODUCT_NAMES["freshservice"]="freshservice"
+PRODUCT_NAMES["freshworks-crm"]="freshworks_crm"
+declare -A PRODUCT_FILE_IDS
+PRODUCT_FILE_IDS["freshdesk"]="staging_fd_external_file_123"
+PRODUCT_FILE_IDS["freshservice"]="staging_fs_external_file_123"
+PRODUCT_FILE_IDS["freshworks-crm"]="staging_crm_external_file_123"
+
+# Per-collection status tracking
+declare -A COLLECTION_STATUS
+declare -A COLLECTION_PASSED
+declare -A COLLECTION_FAILED
+
+# For multi-product, we track per product-collection
+# For single product, we track per collection
+COLLECTION_STATUS["V3_CRUD"]="Not Run"
+COLLECTION_STATUS["V3_FILES"]="Not Run"
+COLLECTION_STATUS["V2"]="Not Run"
+COLLECTION_STATUS["LEGACY"]="Not Run"
+COLLECTION_PASSED["V3_CRUD"]=0
+COLLECTION_PASSED["V3_FILES"]=0
+COLLECTION_PASSED["V2"]=0
+COLLECTION_PASSED["LEGACY"]=0
+COLLECTION_FAILED["V3_CRUD"]=0
+COLLECTION_FAILED["V3_FILES"]=0
+COLLECTION_FAILED["V2"]=0
+COLLECTION_FAILED["LEGACY"]=0
+
+# Multi-product tracking
+declare -A PRODUCT_STATUS
+PRODUCT_STATUS["freshdesk"]="Not Run"
+PRODUCT_STATUS["freshservice"]="Not Run"
+PRODUCT_STATUS["freshworks-crm"]="Not Run"
+
+for arg in "$@"; do
+    case $arg in
+        --v3)
+            RUN_V3=true
+            RUN_ALL=false
+            ;;
+        --v2)
+            RUN_V2=true
+            RUN_ALL=false
+            ;;
+        --legacy)
+            RUN_LEGACY=true
+            RUN_ALL=false
+            ;;
+        --multi-product)
+            MULTI_PRODUCT=true
+            RUN_V3=true
+            RUN_ALL=false
+            ;;
+        --help|-h)
+            SHOW_HELP=true
+            ;;
+    esac
+done
+
+if [ "$SHOW_HELP" = true ]; then
+    head -33 "$0" | tail -30
+    exit 0
+fi
+
+if [ "$RUN_ALL" = true ]; then
+    RUN_V3=true
+    RUN_V2=true
+    # Skip Legacy tests in staging by default - they run locally via WireMock.
+    RUN_LEGACY=false
+fi
+
+echo "========================================="
+if [ "$MULTI_PRODUCT" = true ]; then
+    echo "${REPORT_LOGO} Staging Tests (Multi-Product)"
+else
+    echo "${REPORT_LOGO} Staging Tests"
+fi
+echo "========================================="
+echo ""
+if [ "$MULTI_PRODUCT" = true ]; then
+    echo "Mode: Multi-Product Testing"
+    echo "Products:"
+    echo "  - Freshdesk (product_id: 9)"
+    echo "  - Freshservice (product_id: 3)"
+    echo "  - Freshworks CRM (product_id: 39)"
+    echo ""
+    echo "Test Suites: V3 API (CRUD + Files) for each product"
+else
+    echo "Test Suites:"
+    [ "$RUN_V3" = true ] && echo "  ✓ V3 API (CRUD + Files)"
+    [ "$RUN_V2" = true ] && echo "  ✓ V2 API (OAuth, Transfer)"
+    [ "$RUN_LEGACY" = true ] && echo "  ✓ Legacy API"
+    [ "$RUN_LEGACY" = false ] && echo "  ⊘ Legacy API (skipped)"
+fi
+echo ""
+
+# Check prerequisites
+echo "Checking prerequisites..."
+command -v jq >/dev/null 2>&1 || { echo "Error: jq required" >&2; exit 1; }
+command -v aws >/dev/null 2>&1 || { echo "Error: AWS CLI required" >&2; exit 1; }
+command -v node >/dev/null 2>&1 || { echo "Error: Node.js required" >&2; exit 1; }
+
+if [ ! -f "$CONFIG_FILE" ]; then
+    echo "Error: Config file not found: $CONFIG_FILE"
+    exit 1
+fi
+
+
+echo "✓ Prerequisites OK"
+echo ""
+
+# Load configuration
+echo "========================================="
+echo "Step 1: Loading Configuration"
+echo "========================================="
+AWS_SECRET_NAME=$(jq -r '.aws.secretName' "$CONFIG_FILE")
+AWS_REGION=$(jq -r '.aws.region' "$CONFIG_FILE")
+AWS_CLIENT=$(jq -r '.aws.client' "$CONFIG_FILE")
+AWS_ISSUER=$(jq -r '.aws.issuer' "$CONFIG_FILE")
+BASE_URL=$(jq -r '.base_url' "$CONFIG_FILE")
+echo "  Base URL: $BASE_URL"
+echo ""
+
+# Fetch JWT secret from AWS
+echo "========================================="
+echo "Step 2: Fetching JWT Secret from AWS"
+echo "========================================="
+JWT_SECRET=$("$SCRIPT_DIR/../auth/get-aws-secret.sh" "$AWS_SECRET_NAME" "$AWS_CLIENT" "$AWS_ISSUER" "$AWS_REGION") || {
+    echo "Error: Failed to fetch JWT secret"
+    exit 1
+}
+echo "✓ JWT secret fetched"
+echo ""
+
+# Generate JWT token
+echo "========================================="
+echo "Step 3: Generating JWT Token"
+echo "========================================="
+export JWT_SECRET="$JWT_SECRET"
+export API_ISSUER=$(jq -r '.jwt.api_issuer' "$CONFIG_FILE")
+export API_ACCOUNT_ID=$(jq -r '.jwt.api_account_id' "$CONFIG_FILE")
+export API_ACCOUNT_DOMAIN=$(jq -r '.jwt.api_account_domain' "$CONFIG_FILE")
+export API_USER_ID=$(jq -r '.jwt.api_user_id' "$CONFIG_FILE")
+export API_PRODUCT=$(jq -r '.jwt.api_product' "$CONFIG_FILE")
+export API_PRODUCT_ID=$(jq -r '.jwt.api_product_id' "$CONFIG_FILE")
+export API_USER_ROLE=$(jq -r '.jwt.api_user_role' "$CONFIG_FILE")
+export API_POD=$(jq -r '.jwt.api_pod' "$CONFIG_FILE")
+export API_SKUS=$(jq -r '.jwt.api_skus' "$CONFIG_FILE")
+export SKU=$(jq -r '.jwt.sku' "$CONFIG_FILE")
+export IS_SOURCE_INTERNAL=$(jq -r '.jwt.is_source_internal' "$CONFIG_FILE")
+export ORGANISATION_ID=$(jq -r '.jwt.organisation_id' "$CONFIG_FILE")
+export FRESHID_ACCOUNT_ID=$(jq -r '.jwt.freshid_account_id' "$CONFIG_FILE")
+export API_JWT_EXPIRY=$(jq -r '.jwt.api_jwt_expiry' "$CONFIG_FILE")
+
+JWT_TOKEN=$("$SCRIPT_DIR/../auth/generate-jwt.sh") || {
+    echo "Error: Failed to generate JWT token"
+    exit 1
+}
+unset JWT_SECRET  # Clear secret immediately
+echo "✓ JWT token generated"
+echo ""
+
+# Install Newman
+echo "========================================="
+echo "Step 4: Installing Newman"
+echo "========================================="
+cd "$POSTMAN_DIR"
+npm ci --ignore-scripts --silent 2>/dev/null || npm install --ignore-scripts --silent
+echo "✓ Newman ready"
+echo ""
+
+mkdir -p "$REPORT_DIR"
+mkdir -p "$REPORT_DIR/json"
+
+# Test parameters from config
+EXTENSION_ID=$(jq -r '.test_data.extension_id' "$CONFIG_FILE")
+VERSION_ID=$(jq -r '.test_data.version_id' "$CONFIG_FILE")
+EXTENSION_TYPE=$(jq -r '.test_data.extension_type' "$CONFIG_FILE")
+
+# Initialize results
+TOTAL_PASSED=0
+TOTAL_FAILED=0
+declare -a RESULTS
+
+# Parse Newman output to extract assertion counts
+# Newman outputs a table like: │              assertions │              20 │               0 │
+parse_newman_results() {
+    local output_file="$1"
+    local assertions_line=$(grep "assertions" "$output_file" 2>/dev/null | head -1)
+    
+    NEWMAN_EXECUTED=$(echo "$assertions_line" | awk -F'│' '{gsub(/[^0-9]/,"",$3); print $3}')
+    NEWMAN_FAILED=$(echo "$assertions_line" | awk -F'│' '{gsub(/[^0-9]/,"",$4); print $4}')
+    
+    NEWMAN_EXECUTED=${NEWMAN_EXECUTED:-0}
+    NEWMAN_FAILED=${NEWMAN_FAILED:-0}
+    NEWMAN_PASSED=$((NEWMAN_EXECUTED > 0 ? NEWMAN_EXECUTED - NEWMAN_FAILED : 0))
+}
+
+# Helper function to run all collections in a directory
+run_collection_dir() {
+    local dir_name="$1"
+    local collections_dir="$2"
+    local report_prefix="$3"
+    local collection_key="$4"
+    
+    echo "========================================="
+    echo "Running: $dir_name"
+    echo "========================================="
+    
+    local dir_passed=0
+    local dir_failed=0
+    local all_passed=true
+    
+    # Create a working copy of the environment file to chain variables between collections
+    local working_env="/tmp/newman-staging-env-${collection_key}.json"
+    cp "$ENVIRONMENT" "$working_env"
+    
+    # Run collections in numerical order
+    for collection_file in "$collections_dir"/*.json; do
+        if [ ! -f "$collection_file" ]; then
+            continue
+        fi
+        
+        local basename=$(basename "$collection_file" .json)
+        local report_name="${report_prefix}-${basename}"
+        
+        echo ""
+        echo "  → Running: $basename"
+        echo ""
+        
+        local html_report="$REPORT_DIR/$report_name-staging-$TIMESTAMP.html"
+        local json_report="$REPORT_DIR/json/$report_name-staging-$TIMESTAMP.json"
+        local env_export="/tmp/newman-env-export-staging-${basename}.json"
+        
+        if "$POSTMAN_DIR/node_modules/.bin/newman" run "$collection_file" \
+            --environment "$working_env" \
+            --env-var "auth_token=$JWT_TOKEN" \
+            --env-var "base_url=$BASE_URL" \
+            --env-var "base_url_v2=${BASE_URL%/api/v3}/api/v2" \
+            --env-var "legacy_base_url=${BASE_URL%/api/v3}" \
+            --env-var "extension_id=$EXTENSION_ID" \
+            --env-var "version_id=$VERSION_ID" \
+            --env-var "extension_type=$EXTENSION_TYPE" \
+            --env-var "product_id=$API_PRODUCT_ID" \
+            --env-var "account_id=$API_ACCOUNT_ID" \
+            --env-var "organisation_id=$ORGANISATION_ID" \
+            --env-var "freshid_account_id=$FRESHID_ACCOUNT_ID" \
+            --env-var "user_role=$API_USER_ROLE" \
+            --reporters cli,htmlextra,json \
+            --reporter-htmlextra-export "$html_report" \
+            --reporter-htmlextra-title "${REPORT_LOGO} - $dir_name - $basename (Staging)" \
+            --reporter-htmlextra-darkTheme \
+            --reporter-json-export "$json_report" \
+            --export-environment "$env_export" \
+            --timeout-request 30000 \
+            --delay-request 200 2>&1 | tee /tmp/newman-output.txt; then
+            
+            # Extract pass/fail counts using proper parser
+            parse_newman_results /tmp/newman-output.txt
+            
+            dir_passed=$((dir_passed + NEWMAN_PASSED))
+            dir_failed=$((dir_failed + NEWMAN_FAILED))
+            
+            # Update working environment for next collection (chain variables like installation_id)
+            if [ -f "$env_export" ]; then
+                cp "$env_export" "$working_env"
+            fi
+            
+            echo "  ✓ $basename: PASSED ($NEWMAN_PASSED passed, $NEWMAN_FAILED failed)"
+        else
+            # Extract counts even on failure using proper parser
+            parse_newman_results /tmp/newman-output.txt
+            
+            dir_passed=$((dir_passed + NEWMAN_PASSED))
+            dir_failed=$((dir_failed + NEWMAN_FAILED))
+            all_passed=false
+            
+            # Update working environment even on failure (to preserve any variables set)
+            if [ -f "$env_export" ]; then
+                cp "$env_export" "$working_env"
+            fi
+            
+            echo "  ✗ $basename: FAILED ($NEWMAN_PASSED passed, $NEWMAN_FAILED failed)"
+        fi
+    done
+    
+    # Cleanup temporary environment files
+    rm -f "$working_env" /tmp/newman-env-export-staging-*.json
+    
+    # Update collection status
+    if [ "$all_passed" = true ] && [ "$dir_failed" -eq 0 ]; then
+        COLLECTION_STATUS["$collection_key"]="✓ PASSED"
+        RESULTS+=("$dir_name: ✓ PASSED ($dir_passed passed, $dir_failed failed)")
+        echo ""
+        echo "✓ $dir_name: ALL PASSED"
+    else
+        COLLECTION_STATUS["$collection_key"]="✗ FAILED"
+        RESULTS+=("$dir_name: ✗ FAILED ($dir_passed passed, $dir_failed failed)")
+        echo ""
+        echo "✗ $dir_name: SOME FAILED"
+    fi
+    
+    COLLECTION_PASSED["$collection_key"]=$dir_passed
+    COLLECTION_FAILED["$collection_key"]=$dir_failed
+    TOTAL_PASSED=$((TOTAL_PASSED + dir_passed))
+    TOTAL_FAILED=$((TOTAL_FAILED + dir_failed))
+    
+    [ "$all_passed" = true ] && [ "$dir_failed" -eq 0 ]
+}
+
+# Run collection helper
+run_collection() {
+    local name="$1"
+    local collection="$2"
+    local report_name="$3"
+    local collection_key="$4"  # Key for status tracking
+
+    echo "========================================="
+    echo "Running: $name"
+    echo "========================================="
+
+    local html_report="$REPORT_DIR/$report_name-staging-$TIMESTAMP.html"
+    local json_report="$REPORT_DIR/json/$report_name-staging-$TIMESTAMP.json"
+
+    if "$POSTMAN_DIR/node_modules/.bin/newman" run "$collection" \
+        --environment "$ENVIRONMENT" \
+        --env-var "auth_token=$JWT_TOKEN" \
+        --env-var "base_url=$BASE_URL" \
+        --env-var "base_url_v2=${BASE_URL%/api/v3}/api/v2" \
+        --env-var "legacy_base_url=${BASE_URL%/api/v3}" \
+        --env-var "extension_id=$EXTENSION_ID" \
+        --env-var "version_id=$VERSION_ID" \
+        --env-var "extension_type=$EXTENSION_TYPE" \
+        --env-var "product_id=$API_PRODUCT_ID" \
+        --env-var "account_id=$API_ACCOUNT_ID" \
+        --env-var "organisation_id=$ORGANISATION_ID" \
+        --env-var "freshid_account_id=$FRESHID_ACCOUNT_ID" \
+        --env-var "user_role=$API_USER_ROLE" \
+        --reporters cli,htmlextra,json \
+        --reporter-htmlextra-export "$html_report" \
+        --reporter-htmlextra-title "${REPORT_LOGO} - $name (Staging)" \
+        --reporter-htmlextra-darkTheme \
+        --reporter-json-export "$json_report" \
+        --timeout-request 30000 \
+        --delay-request 200 2>&1 | tee /tmp/newman-output.txt; then
+        
+        # Extract pass/fail counts using proper parser
+        parse_newman_results /tmp/newman-output.txt
+        
+        echo "✓ $name: PASSED"
+        RESULTS+=("$name: ✓ PASSED ($NEWMAN_PASSED passed, $NEWMAN_FAILED failed)")
+        COLLECTION_STATUS["$collection_key"]="✓ PASSED"
+        COLLECTION_PASSED["$collection_key"]=$NEWMAN_PASSED
+        COLLECTION_FAILED["$collection_key"]=$NEWMAN_FAILED
+        TOTAL_PASSED=$((TOTAL_PASSED + NEWMAN_PASSED))
+        TOTAL_FAILED=$((TOTAL_FAILED + NEWMAN_FAILED))
+        return 0
+    else
+        # Extract counts even on failure using proper parser
+        parse_newman_results /tmp/newman-output.txt
+        
+        echo "✗ $name: FAILED"
+        RESULTS+=("$name: ✗ FAILED ($NEWMAN_PASSED passed, $NEWMAN_FAILED failed)")
+        COLLECTION_STATUS["$collection_key"]="✗ FAILED"
+        COLLECTION_PASSED["$collection_key"]=$NEWMAN_PASSED
+        COLLECTION_FAILED["$collection_key"]=$NEWMAN_FAILED
+        TOTAL_PASSED=$((TOTAL_PASSED + NEWMAN_PASSED))
+        TOTAL_FAILED=$((TOTAL_FAILED + NEWMAN_FAILED))
+        return 1
+    fi
+}
+
+# Run tests
+echo ""
+
+if [ "$MULTI_PRODUCT" = true ]; then
+    # Multi-product mode: Run V3 tests for each product
+    for product in $PRODUCTS; do
+        echo ""
+        echo "========================================="
+        echo "Testing Product: $product"
+        echo "========================================="
+        
+        # Use base staging environment, override product-specific vars
+        PRODUCT_JWT_TOKEN="$JWT_TOKEN"
+        PRODUCT_ID="${PRODUCT_IDS[$product]}"
+        PRODUCT_NAME="${PRODUCT_NAMES[$product]}"
+        PRODUCT_FILE_ID="${PRODUCT_FILE_IDS[$product]}"
+
+        PRODUCT_PASSED=true
+
+        # Each product gets its own report subdirectory
+        PRODUCT_REPORT_DIR="$REPORT_DIR/${product}"
+        PRODUCT_JSON_DIR="$REPORT_DIR/json/${product}"
+        mkdir -p "$PRODUCT_REPORT_DIR" "$PRODUCT_JSON_DIR"
+
+        # Working env copy for chaining variables between collections
+        PRODUCT_WORKING_ENV="/tmp/newman-staging-env-${product}-${TIMESTAMP}.json"
+        cp "$ENVIRONMENT" "$PRODUCT_WORKING_ENV"
+
+        # Run CRUD tests for this product
+        crud_key="${product}_CRUD"
+        COLLECTION_STATUS["$crud_key"]="Not Run"
+        COLLECTION_PASSED["$crud_key"]=0
+        COLLECTION_FAILED["$crud_key"]=0
+        CRUD_PASSED=0; CRUD_FAILED=0; CRUD_ALL_PASSED=true
+
+        echo ""
+        echo "Running CRUD API tests for $product..."
+        for collection_file in "$CRUD_COLLECTIONS_DIR"/*.json; do
+            [ ! -f "$collection_file" ] && continue
+            basename=$(basename "$collection_file" .json)
+            REPORT_FILE="$PRODUCT_REPORT_DIR/crud-${basename}-$TIMESTAMP.html"
+            JSON_REPORT="$PRODUCT_JSON_DIR/crud-${basename}-$TIMESTAMP.json"
+            ENV_EXPORT="/tmp/newman-env-export-staging-${product}-crud-${basename}.json"
+            echo "  → $basename"
+            "$POSTMAN_DIR/node_modules/.bin/newman" run "$collection_file" \
+                --environment "$PRODUCT_WORKING_ENV" \
+                --env-var "auth_token=$PRODUCT_JWT_TOKEN" \
+                --env-var "base_url=$BASE_URL" \
+                --env-var "base_url_v2=${BASE_URL%/api/v3}/api/v2" \
+                --env-var "legacy_base_url=${BASE_URL%/api/v3}" \
+                --env-var "extension_id=$EXTENSION_ID" \
+                --env-var "version_id=$VERSION_ID" \
+                --env-var "extension_type=$EXTENSION_TYPE" \
+                --env-var "product_id=$PRODUCT_ID" \
+                --env-var "product_name=$PRODUCT_NAME" \
+                --env-var "file_external_id=$PRODUCT_FILE_ID" \
+                --env-var "account_id=$API_ACCOUNT_ID" \
+                --env-var "organisation_id=$ORGANISATION_ID" \
+                --env-var "freshid_account_id=$FRESHID_ACCOUNT_ID" \
+                --env-var "user_role=$API_USER_ROLE" \
+                --reporters cli,htmlextra,json \
+                --reporter-htmlextra-export "$REPORT_FILE" \
+                --reporter-htmlextra-title "${REPORT_LOGO} - $product - CRUD - $basename (Staging)" \
+                --reporter-htmlextra-darkTheme \
+                --reporter-json-export "$JSON_REPORT" \
+                --export-environment "$ENV_EXPORT" \
+                --timeout-request 30000 \
+                --delay-request 200 2>&1 | tee /tmp/newman-output-${product}.txt
+            parse_newman_results /tmp/newman-output-${product}.txt
+            CRUD_PASSED=$((CRUD_PASSED + NEWMAN_PASSED))
+            CRUD_FAILED=$((CRUD_FAILED + NEWMAN_FAILED))
+            [ -f "$ENV_EXPORT" ] && cp "$ENV_EXPORT" "$PRODUCT_WORKING_ENV"
+            [ "$NEWMAN_FAILED" -gt 0 ] && CRUD_ALL_PASSED=false
+        done
+        COLLECTION_PASSED["$crud_key"]=$CRUD_PASSED
+        COLLECTION_FAILED["$crud_key"]=$CRUD_FAILED
+        TOTAL_PASSED=$((TOTAL_PASSED + CRUD_PASSED))
+        TOTAL_FAILED=$((TOTAL_FAILED + CRUD_FAILED))
+        if [ "$CRUD_ALL_PASSED" = true ]; then
+            echo "✓ CRUD tests passed for $product ($CRUD_PASSED passed, $CRUD_FAILED failed)"
+            COLLECTION_STATUS["$crud_key"]="✓ PASSED"
+        else
+            echo "✗ CRUD tests failed for $product ($CRUD_PASSED passed, $CRUD_FAILED failed)"
+            COLLECTION_STATUS["$crud_key"]="✗ FAILED"
+            PRODUCT_PASSED=false
+        fi
+
+        # Run Files tests for this product
+        files_key="${product}_FILES"
+        COLLECTION_STATUS["$files_key"]="Not Run"
+        COLLECTION_PASSED["$files_key"]=0
+        COLLECTION_FAILED["$files_key"]=0
+        FILES_PASSED=0; FILES_FAILED=0; FILES_ALL_PASSED=true
+
+        echo ""
+        echo "Running Files API tests for $product..."
+        for collection_file in "$FILES_COLLECTIONS_DIR"/*.json; do
+            [ ! -f "$collection_file" ] && continue
+            basename=$(basename "$collection_file" .json)
+            REPORT_FILE="$PRODUCT_REPORT_DIR/files-${basename}-$TIMESTAMP.html"
+            JSON_REPORT="$PRODUCT_JSON_DIR/files-${basename}-$TIMESTAMP.json"
+            ENV_EXPORT="/tmp/newman-env-export-staging-${product}-files-${basename}.json"
+            echo "  → $basename"
+            "$POSTMAN_DIR/node_modules/.bin/newman" run "$collection_file" \
+                --environment "$PRODUCT_WORKING_ENV" \
+                --env-var "auth_token=$PRODUCT_JWT_TOKEN" \
+                --env-var "base_url=$BASE_URL" \
+                --env-var "base_url_v2=${BASE_URL%/api/v3}/api/v2" \
+                --env-var "legacy_base_url=${BASE_URL%/api/v3}" \
+                --env-var "extension_id=$EXTENSION_ID" \
+                --env-var "version_id=$VERSION_ID" \
+                --env-var "extension_type=$EXTENSION_TYPE" \
+                --env-var "product_id=$PRODUCT_ID" \
+                --env-var "product_name=$PRODUCT_NAME" \
+                --env-var "file_external_id=$PRODUCT_FILE_ID" \
+                --env-var "account_id=$API_ACCOUNT_ID" \
+                --env-var "organisation_id=$ORGANISATION_ID" \
+                --env-var "freshid_account_id=$FRESHID_ACCOUNT_ID" \
+                --env-var "user_role=$API_USER_ROLE" \
+                --reporters cli,htmlextra,json \
+                --reporter-htmlextra-export "$REPORT_FILE" \
+                --reporter-htmlextra-title "${REPORT_LOGO} - $product - Files - $basename (Staging)" \
+                --reporter-htmlextra-darkTheme \
+                --reporter-json-export "$JSON_REPORT" \
+                --export-environment "$ENV_EXPORT" \
+                --timeout-request 30000 \
+                --delay-request 200 2>&1 | tee /tmp/newman-output-${product}.txt
+            parse_newman_results /tmp/newman-output-${product}.txt
+            FILES_PASSED=$((FILES_PASSED + NEWMAN_PASSED))
+            FILES_FAILED=$((FILES_FAILED + NEWMAN_FAILED))
+            [ -f "$ENV_EXPORT" ] && cp "$ENV_EXPORT" "$PRODUCT_WORKING_ENV"
+            [ "$NEWMAN_FAILED" -gt 0 ] && FILES_ALL_PASSED=false
+        done
+        rm -f "$PRODUCT_WORKING_ENV" /tmp/newman-env-export-staging-${product}-*.json
+        COLLECTION_PASSED["$files_key"]=$FILES_PASSED
+        COLLECTION_FAILED["$files_key"]=$FILES_FAILED
+        TOTAL_PASSED=$((TOTAL_PASSED + FILES_PASSED))
+        TOTAL_FAILED=$((TOTAL_FAILED + FILES_FAILED))
+        if [ "$FILES_ALL_PASSED" = true ]; then
+            echo "✓ Files tests passed for $product ($FILES_PASSED passed, $FILES_FAILED failed)"
+            COLLECTION_STATUS["$files_key"]="✓ PASSED"
+        else
+            echo "✗ Files tests failed for $product ($FILES_PASSED passed, $FILES_FAILED failed)"
+            COLLECTION_STATUS["$files_key"]="✗ FAILED"
+            PRODUCT_PASSED=false
+        fi
+        
+        # Update product status
+        if [ "$PRODUCT_PASSED" = true ]; then
+            PRODUCT_STATUS["$product"]="✓ PASSED"
+        else
+            PRODUCT_STATUS["$product"]="✗ FAILED"
+        fi
+    done
+else
+    # Single product mode: Run selected test suites
+    if [ "$RUN_V3" = true ]; then
+        run_collection_dir "V3 CRUD API" "$CRUD_COLLECTIONS_DIR" "crud" "V3_CRUD" || true
+        echo ""
+        run_collection_dir "V3 Files API" "$FILES_COLLECTIONS_DIR" "files" "V3_FILES" || true
+        echo ""
+    fi
+
+    if [ "$RUN_V2" = true ]; then
+        run_collection_dir "V2 API" "$V2_COLLECTIONS_DIR" "v2" "V2" || true
+        echo ""
+    fi
+
+    if [ "$RUN_LEGACY" = true ]; then
+        run_collection_dir "Legacy API" "$LEGACY_COLLECTIONS_DIR" "legacy" "LEGACY" || true
+        echo ""
+    fi
+fi
+
+# Summary
+echo ""
+echo "========================================="
+if [ "$MULTI_PRODUCT" = true ]; then
+    echo "Multi-Product Test Results"
+else
+    echo "Staging Test Results"
+fi
+echo "========================================="
+echo ""
+
+if [ "$MULTI_PRODUCT" = true ]; then
+    # Show per-product results
+    echo "Product Results:"
+    for product in $PRODUCTS; do
+        status="${PRODUCT_STATUS[$product]}"
+        crud_key="${product}_CRUD"
+        files_key="${product}_FILES"
+        crud_status="${COLLECTION_STATUS[$crud_key]}"
+        files_status="${COLLECTION_STATUS[$files_key]}"
+        crud_passed="${COLLECTION_PASSED[$crud_key]}"
+        crud_failed="${COLLECTION_FAILED[$crud_key]}"
+        files_passed="${COLLECTION_PASSED[$files_key]}"
+        files_failed="${COLLECTION_FAILED[$files_key]}"
+        
+        # Use display name
+        product_display="${PRODUCT_DISPLAY_NAMES[$product]}"
+        
+        if [ "$status" != "Not Run" ]; then
+            echo "  $product_display: $status"
+            if [ "$crud_status" != "Not Run" ]; then
+                echo "    - CRUD API:  $crud_status ($crud_passed passed, $crud_failed failed)"
+            fi
+            if [ "$files_status" != "Not Run" ]; then
+                echo "    - Files API: $files_status ($files_passed passed, $files_failed failed)"
+            fi
+        fi
+    done
+else
+    # Show per-collection results (original behavior)
+    echo "Collection Results:"
+    for collection_key in "V3_CRUD" "V3_FILES" "V2" "LEGACY"; do
+        status="${COLLECTION_STATUS[$collection_key]}"
+        passed="${COLLECTION_PASSED[$collection_key]}"
+        failed="${COLLECTION_FAILED[$collection_key]}"
+        
+        if [ "$status" != "Not Run" ]; then
+            case $collection_key in
+                V3_CRUD)
+                    echo "  V3 CRUD API:  $status ($passed passed, $failed failed)"
+                    ;;
+                V3_FILES)
+                    echo "  V3 Files API: $status ($passed passed, $failed failed)"
+                    ;;
+                V2)
+                    echo "  V2 API:       $status ($passed passed, $failed failed)"
+                    ;;
+                LEGACY)
+                    echo "  Legacy API:   $status ($passed passed, $failed failed)"
+                    ;;
+            esac
+        fi
+    done
+fi
+
+echo ""
+echo "Overall Total: $TOTAL_PASSED passed, $TOTAL_FAILED failed"
+echo ""
+echo "Reports: $REPORT_DIR/"
+echo ""
+
+# Calculate passed/failed collections
+PASSED_COLLECTIONS=0
+FAILED_COLLECTIONS=0
+if [ "$MULTI_PRODUCT" = true ]; then
+    # Count products as collections for multi-product mode
+    for product in $PRODUCTS; do
+        if [ "${PRODUCT_STATUS[$product]}" = "PASSED" ]; then
+            PASSED_COLLECTIONS=$((PASSED_COLLECTIONS + 1))
+        else
+            FAILED_COLLECTIONS=$((FAILED_COLLECTIONS + 1))
+        fi
+    done
+else
+    # Count individual collections
+    for collection_key in "V3_CRUD" "V3_FILES" "V2" "LEGACY"; do
+        status="${COLLECTION_STATUS[$collection_key]}"
+        if [ "$status" != "Not Run" ]; then
+            if [ "$status" = "✓ PASSED" ]; then
+                PASSED_COLLECTIONS=$((PASSED_COLLECTIONS + 1))
+            else
+                FAILED_COLLECTIONS=$((FAILED_COLLECTIONS + 1))
+            fi
+        fi
+    done
+fi
+
+# Generate consolidated report
+if [ -f "$SCRIPT_DIR/../report-generators/generate-consolidated-report.sh" ]; then
+    echo "========================================="
+    echo "Generating Consolidated Report"
+    echo "========================================="
+    TEST_RESULT=0
+    if [ $TOTAL_FAILED -gt 0 ]; then
+        TEST_RESULT=1
+    fi
+    "$SCRIPT_DIR/../report-generators/generate-consolidated-report.sh" "staging" "$TIMESTAMP" "$REPORT_DIR" "$PASSED_COLLECTIONS" "$FAILED_COLLECTIONS" "$TEST_RESULT" || true
+    echo ""
+fi
+
+# Note: Reports are preserved across runs. Each script manages its own cleanup:
+# - mutation-report.sh: keeps only latest mutation report
+# - java-api-coverage-matrix.sh: keeps only latest coverage matrix
+# - Test reports: kept indefinitely (user can manually clean postman/reports/)
+
+if [ $TOTAL_FAILED -gt 0 ]; then
+    echo "❌ Some tests failed"
+    exit 1
+else
+    echo "✅ All tests passed successfully!"
+fi