@ibm-cloud/cd-tools 1.10.0 → 1.11.1

package/cmd/copy-toolchain.js

@@ -1,6 +1,6 @@
 /**
  * Licensed Materials - Property of IBM
- * (c) Copyright IBM Corporation 2025. All Rights Reserved.
+ * (c) Copyright IBM Corporation 2025, 2026. All Rights Reserved.
  *
  * Note to U.S. Government Users Restricted Rights:
  * Use, duplication or disclosure restricted by GSA ADP Schedule
@@ -19,7 +19,7 @@ import fs from 'node:fs';
 
 import { Command, Option } from 'commander';
 
-import { parseEnvVar } from './utils/utils.js';
+import { parseEnvVar, promptUserConfirmation } from './utils/utils.js';
 import { logger, LOG_STAGES } from './utils/logger.js';
 import { setTerraformEnv, initProviderFile, setupTerraformFiles, runTerraformInit, getNumResourcesPlanned, runTerraformApply, getNumResourcesCreated, getNewToolchainId } from './utils/terraform.js';
 import { getAccountId, getBearerToken, getCdInstanceByRegion, getResourceGroups, getToolchain } from './utils/requests.js';
@@ -110,7 +110,11 @@ async function main(options) {
 		const accountId = await getAccountId(bearer, apiKey);
 
 		// check for continuous delivery instance in target region
-		if (!await getCdInstanceByRegion(bearer, accountId, targetRegion)) throw Error(`Could not find a Continuous Delivery instance in the target region '${targetRegion}', please create one before proceeding.`);
+		if (!await getCdInstanceByRegion(bearer, accountId, targetRegion)) {
+			// give users the option to bypass
+			logger.warn(`Warning! Could not find a Continuous Delivery instance in the target region '${targetRegion}' or you do not have permission to view, please create one before proceeding if one does not exist already.`, LOG_STAGES.setup);
+			await promptUserConfirmation(`Do you want to proceed anyway?`, 'yes', 'Toolchain migration cancelled.');
+		}
 
 		// check for existing .tf files in output directory
 		if (fs.existsSync(outputDir)) {

package/cmd/direct-transfer.js

@@ -11,20 +11,24 @@ import { Command } from 'commander';
 import axios from 'axios';
 import readline from 'readline/promises';
 import { writeFile } from 'fs/promises';
-import { TARGET_REGIONS, SOURCE_REGIONS } from '../config.js';
+import { SOURCE_REGIONS } from '../config.js';
 import { getWithRetry } from './utils/requests.js';
 
 const HTTP_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes default
 
 class GitLabClient {
   constructor(baseURL, token) {
+    const root = baseURL.endsWith("/") ? baseURL : `${baseURL}/`;
+    
     this.client = axios.create({
-      baseURL: baseURL.endsWith('/') ? `${baseURL}api/v4` : `${baseURL}/api/v4`,
+      baseURL: `${root}api/v4`,
       timeout: HTTP_TIMEOUT_MS,
-      headers: {
-        'Authorization': `Bearer ${token}`,
-        'Content-Type': 'application/json'
-      }
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+    });
+    this.graph = axios.create({
+      baseURL: `${root}api`,
+      timeout: HTTP_TIMEOUT_MS,
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
     });
   }
 
@@ -110,6 +114,50 @@ class GitLabClient {
     return projects;
   }
 
+  async listGroupProjectsGraphQL(groupFullPath, { includeSubgroups = true, pageSize = 200, maxProjects = 5000 } = {}) {
+    const out = [];
+    let after = null;
+
+    const query = `
+      query($fullPath: ID!, $after: String, $includeSubgroups: Boolean!, $pageSize: Int!) {
+        group(fullPath: $fullPath) {
+          projects(includeSubgroups: $includeSubgroups, first: $pageSize, after: $after) {
+            nodes {
+              fullPath
+              nameWithNamespace
+              httpUrlToRepo
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+      }
+    `;
+
+    while (out.length < maxProjects) {
+      const resp = await this.graph.post("/graphql", {
+        query,
+        variables: { fullPath: groupFullPath, after, includeSubgroups, pageSize },
+      });
+
+      if (resp.data?.errors?.length) {
+        throw new Error(`GraphQL errors: ${JSON.stringify(resp.data.errors)}`);
+      }
+
+      const projects = resp.data?.data?.group?.projects?.nodes || [];
+      const pageInfo = resp.data?.data?.group?.projects?.pageInfo;
+
+      out.push(...projects);
+
+      if (!pageInfo?.hasNextPage) break;
+      after = pageInfo.endCursor;
+    }
+
+    return out;
+  }
+
   async getGroup(groupId) {
     const response = await this.client.get(`/groups/${groupId}`);
     return response.data;
@@ -186,26 +234,26 @@ class GitLabClient {
     }
   }
 
-    async getBulkImportEntitiesAll(importId, { perPage = 100, maxPages = 200 } = {}) {
-      const all = [];
-      let page = 1;
+  async getBulkImportEntitiesAll(importId, { perPage = 100, maxPages = 200 } = {}) {
+    const all = [];
+    let page = 1;
 
-      while (page <= maxPages) {
-        const resp = await getWithRetry(
-          this.client,
-          `/bulk_imports/${importId}/entities`,
-          { page, per_page: perPage }
-        );
+    while (page <= maxPages) {
+      const resp = await getWithRetry(
+        this.client,
+        `/bulk_imports/${importId}/entities`,
+        { page, per_page: perPage }
+      );
 
-        all.push(...(resp.data || []));
+      all.push(...(resp.data || []));
 
-        const nextPage = Number(resp.headers?.['x-next-page'] || 0);
-        if (!nextPage) break;
+      const nextPage = Number(resp.headers?.['x-next-page'] || 0);
+      if (!nextPage) break;
 
-        page = nextPage;
-      }
+      page = nextPage;
+    }
 
-      return all;
+    return all;
   }
 
   async getGroupByFullPath(fullPath) {
@@ -250,23 +298,19 @@ function validateAndConvertRegion(region) {
 }
 
 //  Build a mapping of: old http_url_to_repo -> new http_url_to_repo
-async function generateUrlMappingFile({sourceUrl, destUrl, sourceGroup, destinationGroupPath, sourceProjects}) {
+async function generateUrlMappingFile({ destUrl, sourceGroup, destinationGroupPath, sourceProjects }) {
   const destBase = destUrl.endsWith('/') ? destUrl.slice(0, -1) : destUrl;
   const urlMapping = {};
 
-  const groupPrefix = `${sourceGroup.full_path}/`;
-
   for (const project of sourceProjects) {
-    const oldRepoUrl = project.http_url_to_repo; // ends with .git
+    const oldRepoUrl = project.http_url_to_repo || project.httpUrlToRepo;
 
-    // path_with_namespace is like "group/subgroup/project-1"
-    let relativePath;
-    if (project.path_with_namespace.startsWith(groupPrefix)) {
-      relativePath = project.path_with_namespace.slice(groupPrefix.length);
-    } else {
-      // Fallback if for some reason full_path is not a prefix
-      relativePath = project.path_with_namespace;
-    }
+    const fullPath = project.path_with_namespace || project.fullPath || "";
+    const groupPrefix = `${sourceGroup.full_path}/`;
+
+    const relativePath = fullPath.startsWith(groupPrefix)
+      ? fullPath.slice(groupPrefix.length)
+      : fullPath;
 
     const newRepoUrl = `${destBase}/${destinationGroupPath}/${relativePath}.git`;
     urlMapping[oldRepoUrl] = newRepoUrl;
@@ -387,7 +431,7 @@ function isGroupEntity(e) {
   return e?.source_type === 'group_entity' || e?.entity_type === 'group_entity' || e?.entity_type === 'group';
 }
 
-async function handleBulkImportConflict({destination, destUrl, sourceGroupFullPath, destinationGroupPath, importResErr}) {
+async function handleBulkImportConflict({ destination, destUrl, sourceGroupFullPath, destinationGroupPath, importResErr }) {
   const historyUrl = buildGroupImportHistoryUrl(destUrl);
   const groupUrl = buildGroupUrl(destUrl, `/groups/${destinationGroupPath}`);
   const fallback = () => {
@@ -463,14 +507,24 @@ async function directTransfer(options) {
     console.log(`Fetching source group from ID: ${options.groupId}...`);
     const sourceGroup = await source.getGroup(options.groupId);
 
-    let destinationGroupName = options.newName || sourceGroup.name;
     let destinationGroupPath = options.newName || sourceGroup.path;
 
-    const sourceProjects = await source.getGroupProjects(sourceGroup.id);
+    let sourceProjects;
+    try {
+      sourceProjects = await source.listGroupProjectsGraphQL(sourceGroup.full_path, {
+        includeSubgroups: true,
+        pageSize: 100,
+        maxProjects: 10000,
+      });
+    } catch (e) {
+      console.warn(`[WARN] GraphQL listing failed (${e.message}). Falling back to REST safe listing...`);
+      sourceProjects = await source.getGroupProjects(sourceGroup.id);
+    }
+    
     console.log(`Found ${sourceProjects.length} projects in source group`);
     if (sourceProjects.length > 0) {
       console.log('Projects to be migrated:');
-      sourceProjects.forEach(p => console.log(`${p.name_with_namespace}`));
+      sourceProjects.forEach(p => console.log(p.name_with_namespace || p.nameWithNamespace || p.fullPath));
     }
 
     if (options.newName) {
@@ -479,7 +533,6 @@ async function directTransfer(options) {
 
     // Generate URL mapping JSON before starting the migration
     await generateUrlMappingFile({
-      sourceUrl,
       destUrl,
       sourceGroup,
       destinationGroupPath,

package/cmd/export-secrets.js

@@ -56,7 +56,7 @@ async function main(options) {
             const decomposedCrn = decomposeCrn(toolchainCrn);
             toolchainId = decomposedCrn.serviceInstance;
             region = decomposedCrn.location;
-        } catch (e) {
+        } catch {
             throw Error('Provided toolchain CRN is invalid');
         }
 

package/cmd/utils/logger.js

@@ -107,12 +107,7 @@ class Logger {
 
     async withSpinner(asyncFn, loadingMsg, successMsg, prefix, ...args) {
         if (this.verbosity < 1 || DISABLE_SPINNER) {
-            try {
-                return await asyncFn(...args);
-            }
-            catch (err) {
-                throw (err);
-            }
+            return await asyncFn(...args);
         }
 
         this.spinner = ora({

package/cmd/utils/requests.js

@@ -327,10 +327,11 @@ async function getGritUserProject(privToken, region, user, projectName) {
     };
     const response = await axios(options);
     switch (response.status) {
-        case 200:
+        case 200: {
             const found = response.data?.find((entry) => entry['path'] === projectName);
             if (!found) throw Error('GRIT user project not found');
             return;
+        }
         default:
             throw Error('Get GRIT user project failed');
     }
@@ -348,10 +349,11 @@ async function getGritGroup(privToken, region, groupName) {
     };
     const response = await axios(options);
     switch (response.status) {
-        case 200:
+        case 200: {
             const found = response.data?.find((entry) => entry['full_path'] === groupName);
             if (!found) throw Error('GRIT group not found');
             return found['id'];
+        }
         default:
             throw Error('Get GRIT group failed');
     }
@@ -370,10 +372,11 @@ async function getGritGroupProject(privToken, region, groupId, projectName) {
     };
     const response = await axios(options);
     switch (response.status) {
-        case 200:
+        case 200: {
             const found = response.data?.find((entry) => entry['path'] === projectName);
             if (!found) throw Error('GRIT group project not found');
             return;
+        }
         default:
             throw Error('Get GRIT group project failed');
     }
@@ -479,24 +482,24 @@ async function migrateToolchainSecrets(bearer, data, region) {
 
 // GET with retry for flaky 5xx/520 errors (Cloudflare / origin issues)
 async function getWithRetry(client, path, params = {}, { retries = 3, retryDelayMs = 2000 } = {}) {
-  let lastError;
-  for (let attempt = 1; attempt <= retries; attempt++) {
-    try {
-      return await client.get(path, { params });
-    } catch (error) {
-      const status = error.response?.status;
-      if (attempt < retries && status && status >= 500) {
-        console.warn(
-          `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
-        );
-        await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
-        lastError = error;
-        continue;
-      }
-      throw error; // Non-5xx or out of retries: rethrow
+    let lastError;
+    for (let attempt = 1; attempt <= retries; attempt++) {
+        try {
+            return await client.get(path, { params });
+        } catch (error) {
+            const status = error.response?.status;
+            if (attempt < retries && status && status >= 500) {
+                console.warn(
+                    `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
+                );
+                await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
+                lastError = error;
+                continue;
+            }
+            throw error; // Non-5xx or out of retries: rethrow
+        }
     }
-  }
-  throw lastError;
+    throw lastError;
 }
 
 export {

package/cmd/utils/terraform.js

@@ -31,7 +31,7 @@ const writeFilePromise = promisify(fs.writeFile)
 async function execPromise(command, options) {
     try {
         const exec = promisify(child_process.exec);
-        const { stdout, stderr } = await exec(command, options);
+        const { stdout, _ } = await exec(command, options);
         return stdout.trim();
     } catch (err) {
         throw new Error(`Command failed: ${command} \n${err.stderr || err.stdout}`);
@@ -419,7 +419,6 @@ async function runTerraformApply(skipTfConfirmation, outputDir, verbosity, targe
     });
 
     let stdoutData = '';
-    let stderrData = '';
 
     child.stdout.on('data', (chunk) => {
         const text = chunk.toString();
@@ -432,7 +431,6 @@ async function runTerraformApply(skipTfConfirmation, outputDir, verbosity, targe
 
     child.stderr.on('data', (chunk) => {
         const text = chunk.toString();
-        stderrData += text;
         if (verbosity >= 1) {
             process.stderr.write(text);
             logger.dump(text);

package/cmd/utils/validate.js

@@ -86,8 +86,7 @@ async function warnDuplicateName(token, accountId, tcName, srcRegion, targetRegi
     const toolchains = await getToolchainsByName(token, accountId, tcName);
 
     let hasSameRegion = false;
-    let hasSameResourceGroup = false;
-    let hasBoth = false;
+    let hasBoth = false; // same region and resource group
 
     if (toolchains.length > 0) {
         let newTcName = tcName;
@@ -100,8 +99,6 @@ async function warnDuplicateName(token, accountId, tcName, srcRegion, targetRegi
                 } else {
                     hasSameRegion = true;
                 }
-            } else if (tc.resource_group_id === targetResourceGroupId) {
-                hasSameResourceGroup = true;
             }
         });
 
@@ -117,10 +114,6 @@ async function warnDuplicateName(token, accountId, tcName, srcRegion, targetRegi
                 // soft warning of confusion
                 logger.warn(`\nWarning! A toolchain named '${tcName}' already exists in:\n - Region: ${targetRegion}`, '', true);
             }
-            // if (hasSameResourceGroup) {
-            //     // soft warning of confusion
-            //     logger.warn(`\nWarning! A toolchain named '${tcName}' already exists in:\n - Resource Group: ${targetResourceGroupName} (${targetResourceGroupId})`, '', true);
-            // }
         }
 
         if (hasBoth || hasSameRegion) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.10.0",
+  "version": "1.11.1",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",

package/test/copy-toolchain/functionalities.test.js

@@ -1,6 +1,6 @@
 /**
  * Licensed Materials - Property of IBM
- * (c) Copyright IBM Corporation 2025. All Rights Reserved.
+ * (c) Copyright IBM Corporation 2025, 2026. All Rights Reserved.
  *
  * Note to U.S. Government Users Restricted Rights:
  * Use, duplication or disclosure restricted by GSA ADP Schedule
@@ -15,7 +15,7 @@ import { expect, assert } from 'chai';
 
 import { assertPtyOutput, assertExecError, areFilesInDir, deleteCreatedToolchains, parseTcIdAndRegion } from '../utils/testUtils.js';
 import { getBearerToken, getToolchain } from '../../cmd/utils/requests.js';
-import { TEST_TOOLCHAINS, DEFAULT_RG_ID, R2R_CLI_RG_ID } from '../data/test-toolchains.js';
+import { TEST_TOOLCHAINS, DEFAULT_RG_ID } from '../data/test-toolchains.js';
 import { TARGET_REGIONS } from '../../config.js';
 
 nconf.env('__');
@@ -53,16 +53,17 @@ describe('copy-toolchain: Test functionalities', function () {
                 timeout: 10000
             }
         },
-        {
-            name: 'Check if CD instance exists in target region',
-            cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
-            expected: new RegExp(`Could not find a Continuous Delivery instance in the target region '${TARGET_REGIONS[0]}', please create one before proceeding.`),
-            options: {
-                exitCondition: `Could not find a Continuous Delivery instance in the target region '${TARGET_REGIONS[0]}', please create one before proceeding.`,
-                timeout: 10000,
-                env: { ...process.env, MOCK_ALL_REQUESTS: 'true', MOCK_GET_CD_INSTANCE_BY_REGION_SCENARIO: 'NOT_FOUND' }
-            }
-        },
+        // TODO: update outdated test from when missing cd instance would fail the command
+        // {
+        //     name: 'Check if CD instance exists in target region',
+        //     cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TARGET_REGIONS[0]],
+        //     expected: new RegExp(`Could not find a Continuous Delivery instance in the target region '${TARGET_REGIONS[0]}', please create one before proceeding.`),
+        //     options: {
+        //         exitCondition: `Could not find a Continuous Delivery instance in the target region '${TARGET_REGIONS[0]}', please create one before proceeding.`,
+        //         timeout: 10000,
+        //         env: { ...process.env, MOCK_ALL_REQUESTS: 'true', MOCK_GET_CD_INSTANCE_BY_REGION_SCENARIO: 'NOT_FOUND' }
+        //     }
+        // },
         {
             name: 'Log file is created successfully',
             cmd: [CLI_PATH, COMMAND, '-c', TEST_TOOLCHAINS['empty'].crn, '-r', TEST_TOOLCHAINS['empty'].region],