@ibm-cloud/cd-tools 1.6.0 → 1.7.0

package/README.md

@@ -132,9 +132,9 @@ Copies a toolchain, including tool integrations and Tekton pipelines, to another
 
 Examples:
   export IBMCLOUD_API_KEY='...'
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r us-south
+  npx @ibm-cloud/cd-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r us-south
       Copy a toolchain to the Dallas region with the same name, in the same resource group.
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey ${APIKEY}
+  npx @ibm-cloud/cd-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey ${APIKEY}
       Copy a toolchain to the Frankfurt region with the specified name and target resource group, using the given API key
 
 Environment Variables:

package/cmd/copy-toolchain.js

@@ -322,10 +322,7 @@ async function main(options) {
 			}
 
 			// create toolchain, which invokes script to create s2s if applicable
-			await runTerraformApply(true, outputDir, verbosity, `ibm_cd_toolchain.${toolchainTfName}`).catch((err) => {
-				logger.error(err, LOG_STAGES.tf);
-				applyErrors = true;
-			});
+			await runTerraformApply(true, outputDir, verbosity, `ibm_cd_toolchain.${toolchainTfName}`);
 
 			// create the rest
 			await runTerraformApply(skipUserConfirmation, outputDir, verbosity).catch((err) => {

package/cmd/direct-transfer.js

@@ -7,15 +7,19 @@
  * Contract with IBM Corp.
  */
 
-import { Command, Option } from 'commander';
+import { Command } from 'commander';
 import axios from 'axios';
 import readline from 'readline/promises';
+import { writeFile } from 'fs/promises';
 import { TARGET_REGIONS, SOURCE_REGIONS } from '../config.js';
 
+const HTTP_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes default
+
 class GitLabClient {
   constructor(baseURL, token) {
     this.client = axios.create({
       baseURL: baseURL.endsWith('/') ? `${baseURL}api/v4` : `${baseURL}/api/v4`,
+      timeout: HTTP_TIMEOUT_MS,
       headers: {
         'Authorization': `Bearer ${token}`,
         'Content-Type': 'application/json'
@@ -23,24 +27,110 @@ class GitLabClient {
     });
   }
 
-  async getGroupProjects(groupId) {
+  // List all projects in a group + all its subgroups using BFS.
+  async getGroupProjects(groupId, { maxProjects = 1000, maxRequests = 2000 } = {}) {
+    let requestCount = 0;
     const projects = [];
-    let page = 1;
-    let hasMore = true;
-
-    while (hasMore) {
-      const response = await this.client.get(`/groups/${groupId}/projects`, {
-        params: { page, per_page: 100, include_subgroups: true }
-      });
-      
-      projects.push(...response.data);
-      hasMore = response.data.length === 100;
-      page++;
+    const toVisit = [groupId];
+    const visited = new Set();
+
+    console.log(
+      `[DEBUG] Starting BFS project listing from group ${groupId} (maxProjects=${maxProjects}, maxRequests=${maxRequests})`
+    );
+
+    while (toVisit.length > 0) {
+      const currentGroupId = toVisit.shift();
+      if (visited.has(currentGroupId)) continue;
+      visited.add(currentGroupId);
+
+      console.log(`[DEBUG] Visiting group ${currentGroupId}. Remaining groups in queue: ${toVisit.length}`);
+
+      // List projects for THIS group (no include_subgroups!)
+      let projPage = 1;
+      let hasMoreProjects = true;
+
+      while (hasMoreProjects) {
+        if (requestCount >= maxRequests || projects.length >= maxProjects) {
+          console.warn(`[WARN] Stopping project traversal: requestCount=${requestCount}, projects=${projects.length}`);
+          return projects;
+        }
+
+        const projRes = await this.getWithRetry(
+          `/groups/${currentGroupId}/projects`,
+          { page: projPage, per_page: 100 }
+        );
+
+        requestCount++;
+        const pageProjects = projRes.data || [];
+        if (pageProjects.length > 0) {
+          projects.push(...pageProjects);
+        }
+
+        hasMoreProjects = pageProjects.length === 100;
+        projPage++;
+      }
+
+      // List DIRECT subgroups and enqueue them
+      let subgroupPage = 1;
+      let hasMoreSubgroups = true;
+
+      while (hasMoreSubgroups) {
+        if (requestCount >= maxRequests) {
+          console.warn(
+            `[WARN] Stopping subgroup traversal: requestCount=${requestCount}`
+          );
+          return projects;
+        }
+
+        const subgroupRes = await this.getWithRetry(
+          `/groups/${currentGroupId}/subgroups`,
+          { page: subgroupPage, per_page: 100 }
+        );
+
+        requestCount++;
+        const subgroups = subgroupRes.data || [];
+
+        if (subgroups.length > 0) {
+          for (const sg of subgroups) {
+            if (!visited.has(sg.id)) {
+              toVisit.push(sg.id);
+            }
+          }
+        }
+
+        hasMoreSubgroups = subgroups.length === 100;
+        subgroupPage++;
+      }
     }
-    
+
+    console.log(`[DEBUG] Finished BFS project listing. Total projects=${projects.length}, total requests=${requestCount}`);
     return projects;
   }
-   
+
+  // Helper: GET with retry for flaky 5xx/520 errors (Cloudflare / origin issues)
+  async getWithRetry(path, params = {}, { retries = 3, retryDelayMs = 2000 } = {}) {
+    let lastError;
+    for (let attempt = 1; attempt <= retries; attempt++) {
+      try {
+        return await this.client.get(path, { params });
+      } catch (error) {
+        const status = error.response?.status;
+
+        if (attempt < retries && status && status >= 500) {
+          console.warn(
+            `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
+          );
+          await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
+          lastError = error;
+          continue;
+        }
+
+        throw error; // Non-5xx or out of retries: rethrow
+      }
+    }
+    throw lastError;
+  }
+
   async getGroup(groupId) {
     const response = await this.client.get(`/groups/${groupId}`);
     return response.data;
@@ -126,7 +216,7 @@ async function promptUser(name) {
 
   const answer = await rl.question(`Your new group name is ${name}. Are you sure? (Yes/No)`);
 
-  rl.close(); 
+  rl.close();
 
   if (answer.toLowerCase() === 'yes' || answer.toLowerCase() === 'y') {
     console.log("Proceeding...");
@@ -144,6 +234,39 @@ function validateAndConvertRegion(region) {
   return `https://${region}.git.cloud.ibm.com/`;
 }
 
+//  Build a mapping of: old http_url_to_repo -> new http_url_to_repo and old web_url -> new web_url
+async function generateUrlMappingFile({sourceUrl, destUrl, sourceGroup, destinationGroupPath, sourceProjects}) {
+  const destBase = destUrl.endsWith('/') ? destUrl.slice(0, -1) : destUrl;
+  const urlMapping = {};
+
+  const groupPrefix = `${sourceGroup.full_path}/`;
+
+  for (const project of sourceProjects) {
+    const oldRepoUrl = project.http_url_to_repo; // ends with .git
+
+    // path_with_namespace is like "group/subgroup/project-1"
+    let relativePath;
+    if (project.path_with_namespace.startsWith(groupPrefix)) {
+      relativePath = project.path_with_namespace.slice(groupPrefix.length);
+    } else {
+      // Fallback if for some reason full_path is not a prefix
+      relativePath = project.path_with_namespace;
+    }
+
+    const newRepoUrl = `${destBase}/${destinationGroupPath}/${relativePath}.git`;
+    urlMapping[oldRepoUrl] = newRepoUrl;
+  }
+
+  const mappingFile = 'grit-url-map.json';
+
+  await writeFile(mappingFile, JSON.stringify(urlMapping, null, 2), {
+    encoding: 'utf8',
+  });
+
+  console.log(`\nURL mapping JSON generated at: ${mappingFile}`);
+  console.log(`Total mapped projects: ${sourceProjects.length}`);
+}
+
 async function directTransfer(options) {
   const sourceUrl = validateAndConvertRegion(options.sourceRegion);
   const destUrl = validateAndConvertRegion(options.destRegion);
@@ -168,6 +291,15 @@ async function directTransfer(options) {
       await promptUser(options.newName);
     }
 
+    // Generate URL mapping JSON before starting the migration
+    await generateUrlMappingFile({
+      sourceUrl,
+      destUrl,
+      sourceGroup,
+      destinationGroupPath,
+      sourceProjects,
+    });
+
     let bulkImport = null;
 
     const requestPayload = {
@@ -181,10 +313,10 @@ async function directTransfer(options) {
         destination_slug: destinationGroupPath,
         destination_namespace: ""
       }]
-    }
+    };
 
     let importRes = null;
-    
+
     try {
       importRes = await destination.bulkImport(requestPayload);
       if (importRes.success) {
@@ -192,9 +324,9 @@ async function directTransfer(options) {
         console.log(`Bulk import request succeeded!`);
         console.log(`Bulk import initiated successfully (ID: ${importRes.data?.id})`);
       } else if (importRes.conflict) {
-      console.log(`Conflict detected: ${importRes.error}`);
-      console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
-      process.exit(0);
+        console.log(`Conflict detected: ${importRes.error}`);
+        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
+        process.exit(0);
       }
     } catch (error) {
       console.log(`Bulk import request failed - ${error.message}`);
@@ -204,11 +336,11 @@ async function directTransfer(options) {
     console.log('\nPolling bulk import status (checking every 5 minute)...');
     let importStatus = 'created';
     let attempts = 0;
-    
+
     while (!['finished', 'failed', 'timeout'].includes(importStatus) && attempts < 60) {
       if (attempts > 0) {
         console.log(`Waiting 5 minute before next status check...`);
-        await new Promise(resolve => setTimeout(resolve, 5*60000));
+        await new Promise(resolve => setTimeout(resolve, 5 * 60000));
       }
       try {
         const importDetails = await destination.getBulkImport(bulkImport.id);
@@ -230,7 +362,7 @@ async function directTransfer(options) {
       }
       attempts++;
     }
-    
+
     if (attempts >= 60) {
       console.error(`Bulk import either timed out or is still running in the background`);
       process.exit(0);
@@ -239,20 +371,20 @@ async function directTransfer(options) {
     const entities = await destination.getBulkImportEntities(bulkImport.id);
     const finishedEntities = entities.filter(e => e.status === 'finished');
     const failedEntities = entities.filter(e => e.status === 'failed');
-    
+
     if (importStatus === 'finished' && finishedEntities.length > 0) {
       console.log(`\nGroup migration completed successfully!`);
       console.log(`Migration Results:`);
       console.log(`Successfully migrated: ${finishedEntities.length} entities`);
       console.log(`Failed: ${failedEntities.length} entities`);
-      
+
       if (failedEntities.length > 0) {
         console.log(`\nFailed entities:\n`);
         failedEntities.forEach(e => {
           console.log(`${e.source_type}: ${e.source_full_path} (${e.status})`);
         });
       }
-      
+
       return 0;
     } else {
       console.error('\nBulk import failed!');
@@ -282,7 +414,7 @@ const command = new Command('copy-project-group')
   .showHelpAfterError()
   .hook('preAction', cmd => cmd.showHelpAfterError(false)) // only show help during validation
   .action(async (options) => {
-    await directTransfer(options);  
+    await directTransfer(options);
   });
 
 export default command;

package/cmd/utils/terraform.js

@@ -192,7 +192,7 @@ async function setupTerraformFiles({ token, srcRegion, targetRegion, targetTag,
                             // prompt user
                             const validateGritUrlPrompt = async (str) => {
                                 if (!str) {
-                                    logger.print('Skipping... (URL will remain unchanged in the generatedTerraform configuration)');
+                                    logger.print('Skipping... (URL will remain unchanged in the generated Terraform configuration)');
                                     return '';
                                 }
                                 const newUrl = (GIT_BASE_URL || `https://${targetRegion}.git.cloud.ibm.com`) + `/${str}.git`;
@@ -492,6 +492,7 @@ function replaceDependsOn(str) {
 function addS2sScriptToToolchainTf(str) {
     const provisionerStr = (tfName) => `\n\n  provisioner "local-exec" {
     command = "node create-s2s-script.cjs"
+    on_failure = fail
     environment = {
       IBMCLOUD_API_KEY = var.ibmcloud_api_key
       TARGET_TOOLCHAIN_ID = ibm_cd_toolchain.${tfName}.id

package/config.js

@@ -11,9 +11,9 @@ const COPY_TOOLCHAIN_DESC = `Copies a toolchain, including tool integrations and
 
 Examples:
   export IBMCLOUD_API_KEY='...'
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r us-south
+  npx @ibm-cloud/cd-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r us-south
       Copy a toolchain to the Dallas region with the same name, in the same resource group.
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey \${APIKEY}
+  npx @ibm-cloud/cd-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey \${APIKEY}
       Copy a toolchain to the Frankfurt region with the specified name and target resource group, using the given API key
 
 Environment Variables:

package/create-s2s-script.js

@@ -9,6 +9,7 @@
 
 const fs = require('node:fs');
 const { resolve } = require('node:path');
+const { exit } = require('node:process');
 
 const API_KEY = process.env['IBMCLOUD_API_KEY'];
 if (!API_KEY) throw Error(`Missing 'IBMCLOUD_API_KEY'`);
@@ -103,21 +104,29 @@ async function createS2sAuthPolicy(bearer, item) {
         });
 
         if (!response.ok) {
-            throw new Error(`Response status: ${response.status}, ${response.statusText}`);
+            return Promise.reject(`Failed to create service-to-service authorization policy for ${item['serviceId']} '${item['parameters']['label'] ?? item['parameters']['name']}' with status: ${response.status} ${response.statusText}`);
         }
 
         console.log(`CREATING AUTH POLICY... ${response.status}, ${response.statusText}`);
     } catch (error) {
-        console.error(error.message);
+        return Promise.reject(error.message);
     }
 }
 
 // main
 
-getBearer().then((bearer) => {
+getBearer().then(async (bearer) => {
     const inputArr = JSON.parse(fs.readFileSync(resolve(INPUT_PATH)));
 
-    inputArr.forEach(async (item) => {
-        await createS2sAuthPolicy(bearer, item);
+    const promises = [];
+    inputArr.forEach((item) => {
+        promises.push(createS2sAuthPolicy(bearer, item));
     });
+
+    try {
+        await Promise.all(promises);
+    } catch (e) {
+        console.error(e)
+        exit(1);
+    }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.6.0",
+  "version": "1.7.0",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",

package/test/copy-toolchain/functionalities.test.js

@@ -37,10 +37,19 @@ describe('copy-toolchain: Test functionalities', function () {
     const testCases = [
         {
             name: 'Terraform Version Verification',
-            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
+            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region],
             expected: /✔ Terraform Version:/,
             options: {
-                exitCondition: '(Recommended) Add a tag to the cloned toolchain (Ctrl-C to abort):',
+                exitCondition: `(Recommended) Edit the cloned toolchain's name [default: ${TEST_TOOLCHAINS['empty'].name}] (Ctrl-C to abort):`,
+                timeout: 10000
+            }
+        },
+        {
+            name: 'CLI Version Verification',
+            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region],
+            expected: /✔ cd-tools Version:/,
+            options: {
+                exitCondition: `(Recommended) Edit the cloned toolchain's name [default: ${TEST_TOOLCHAINS['empty'].name}] (Ctrl-C to abort):`,
                 timeout: 10000
             }
         },
@@ -56,10 +65,10 @@ describe('copy-toolchain: Test functionalities', function () {
         },
         {
             name: 'Log file is created successfully',
-            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
+            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region],
             expected: null,
             options: {
-                exitCondition: '(Recommended) Add a tag to the cloned toolchain (Ctrl-C to abort):',
+                exitCondition: `(Recommended) Edit the cloned toolchain's name [default: ${TEST_TOOLCHAINS['empty'].name}] (Ctrl-C to abort):`,
                 timeout: 10000,
                 cwd: TEMP_DIR + '/' + 'log-file-is-created-successfully'
             },
@@ -81,15 +90,6 @@ describe('copy-toolchain: Test functionalities', function () {
                 assert.isTrue(toolchainData.id === toolchainId, 'Was toolchain created successfully without any confirmations?');
             }
         },
-        {
-            name: 'Prompt User when toolchain name already exists in resource group',
-            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
-            expected: new RegExp(`Warning! A toolchain named \'${TEST_TOOLCHAINS['empty'].name}\' already exists in:[\\s\\S]*?Resource Group:[\\s\\S]*?${R2R_CLI_RG_ID}`),
-            options: {
-                exitCondition: '(Recommended) Add a tag to the cloned toolchain (Ctrl-C to abort):',
-                timeout: 10000
-            }
-        },
         {
             name: 'Prompt User when toolchain name already exists in region',
             cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region, '-g', DEFAULT_RG_ID],

package/test/copy-toolchain/input-validation.test.js

@@ -125,10 +125,13 @@ describe('copy-toolchain: Test user input handling', function () {
     const invalidUserInputCases = [
         {
             name: 'Invalid Toolchain tag is provided',
-            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
+            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region],
             expected: /Provided tag is invalid/,
             options: {
-                questionAnswerMap: { '(Recommended) Add a tag to the cloned toolchain (Ctrl-C to abort):': mocks.invalidTag },
+                questionAnswerMap: { 
+                    [`(Recommended) Edit the cloned toolchain's name [default: ${TEST_TOOLCHAINS['empty'].name}] (Ctrl-C to abort):`]: '',
+                    '(Recommended) Add a tag to the cloned toolchain (Ctrl-C to abort):': mocks.invalidTag
+                },
                 exitCondition: 'Validation failed',
                 timeout: 10000
             }

package/test/copy-toolchain/tf-import.test.js

@@ -43,12 +43,12 @@ describe('copy-toolchain: Test import-terraform output', function () {
                     ibm_cd_tekton_pipeline_trigger: 1,
                     ibm_cd_tekton_pipeline_trigger_property: 1,
                     ibm_cd_toolchain: 1,
+                    ibm_cd_toolchain_tool_cos: 1,
                     ibm_cd_toolchain_tool_custom: 1,
                     ibm_cd_toolchain_tool_githubconsolidated: 1,
                     ibm_cd_toolchain_tool_pipeline: 1,
                     ibm_cd_toolchain_tool_secretsmanager: 1,
                     ibm_cd_toolchain_tool_slack: 1,
-                    ibm_iam_authorization_policy: 1
                 });
             }
         },
@@ -68,12 +68,12 @@ describe('copy-toolchain: Test import-terraform output', function () {
                     ibm_cd_tekton_pipeline_trigger: 1,
                     ibm_cd_tekton_pipeline_trigger_property: 1,
                     ibm_cd_toolchain: 1,
+                    ibm_cd_toolchain_tool_cos: 1,
                     ibm_cd_toolchain_tool_custom: 1,
                     ibm_cd_toolchain_tool_githubconsolidated: 1,
                     ibm_cd_toolchain_tool_pipeline: 1,
                     ibm_cd_toolchain_tool_secretsmanager: 1,
                     ibm_cd_toolchain_tool_slack: 1,
-                    ibm_iam_authorization_policy: 1
                 });
             }
         },
@@ -99,7 +99,6 @@ describe('copy-toolchain: Test import-terraform output', function () {
                     ibm_cd_toolchain_tool_pipeline: 1,
                     ibm_cd_toolchain_tool_secretsmanager: 1,
                     ibm_cd_toolchain_tool_slack: 1,
-                    ibm_iam_authorization_policy: 1
                 })
             }
         },

package/test/copy-toolchain/tool-validation.test.js

@@ -56,7 +56,6 @@ describe('copy-toolchain: Test tool validation', function () {
             },
             assertionFunc: (output) => {
                 expect(output).to.match(/Warning! The following tools contain secrets that cannot be migrated/);
-                expect(output).to.match(/cloudobjectstorage[\s\S]*?cos_api_key/);
                 expect(output).to.match(/slack[\s\S]*?api_token/);
                 expect(output).to.match(/pipeline[\s\S]*?properties.doi-ibmcloud-api-key/);
             }