@ibm-cloud/cd-tools 1.6.1 → 1.8.1

package/README.md

@@ -132,9 +132,9 @@ Copies a toolchain, including tool integrations and Tekton pipelines, to another
 
 Examples:
   export IBMCLOUD_API_KEY='...'
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r us-south
+  npx @ibm-cloud/cd-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r us-south
       Copy a toolchain to the Dallas region with the same name, in the same resource group.
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey ${APIKEY}
+  npx @ibm-cloud/cd-tools copy-toolchain -c ${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey ${APIKEY}
       Copy a toolchain to the Frankfurt region with the specified name and target resource group, using the given API key
 
 Environment Variables:

package/cmd/direct-transfer.js

@@ -7,15 +7,20 @@
  * Contract with IBM Corp.
  */
 
-import { Command, Option } from 'commander';
+import { Command } from 'commander';
 import axios from 'axios';
 import readline from 'readline/promises';
+import { writeFile } from 'fs/promises';
 import { TARGET_REGIONS, SOURCE_REGIONS } from '../config.js';
+import { getWithRetry } from './utils/requests.js';
+
+const HTTP_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes default
 
 class GitLabClient {
   constructor(baseURL, token) {
     this.client = axios.create({
       baseURL: baseURL.endsWith('/') ? `${baseURL}api/v4` : `${baseURL}/api/v4`,
+      timeout: HTTP_TIMEOUT_MS,
       headers: {
         'Authorization': `Bearer ${token}`,
         'Content-Type': 'application/json'
@@ -23,24 +28,88 @@ class GitLabClient {
     });
   }
 
-  async getGroupProjects(groupId) {
+  // List all projects in a group + all its subgroups using BFS.
+  async getGroupProjects(groupId, { maxProjects = 1000, maxRequests = 2000 } = {}) {
+    let requestCount = 0;
     const projects = [];
-    let page = 1;
-    let hasMore = true;
-
-    while (hasMore) {
-      const response = await this.client.get(`/groups/${groupId}/projects`, {
-        params: { page, per_page: 100, include_subgroups: true }
-      });
-      
-      projects.push(...response.data);
-      hasMore = response.data.length === 100;
-      page++;
+    const toVisit = [groupId];
+    const visited = new Set();
+
+    console.log(
+      `[DEBUG] Starting BFS project listing from group ${groupId} (maxProjects=${maxProjects}, maxRequests=${maxRequests})`
+    );
+
+    while (toVisit.length > 0) {
+      const currentGroupId = toVisit.shift();
+      if (visited.has(currentGroupId)) continue;
+      visited.add(currentGroupId);
+
+      console.log(`[DEBUG] Visiting group ${currentGroupId}. Remaining groups in queue: ${toVisit.length}`);
+
+      // List projects for THIS group (no include_subgroups!)
+      let projPage = 1;
+      let hasMoreProjects = true;
+
+      while (hasMoreProjects) {
+        if (requestCount >= maxRequests || projects.length >= maxProjects) {
+          console.warn(`[WARN] Stopping project traversal: requestCount=${requestCount}, projects=${projects.length}`);
+          return projects;
+        }
+
+        const projRes = await getWithRetry(
+          this.client,
+          `/groups/${currentGroupId}/projects`,
+          { page: projPage, per_page: 100 }
+        );
+
+        requestCount++;
+        const pageProjects = projRes.data || [];
+        if (pageProjects.length > 0) {
+          projects.push(...pageProjects);
+        }
+
+        hasMoreProjects = pageProjects.length === 100;
+        projPage++;
+      }
+
+      // List DIRECT subgroups and enqueue them
+      let subgroupPage = 1;
+      let hasMoreSubgroups = true;
+
+      while (hasMoreSubgroups) {
+        if (requestCount >= maxRequests) {
+          console.warn(
+            `[WARN] Stopping subgroup traversal: requestCount=${requestCount}`
+          );
+          return projects;
+        }
+
+        const subgroupRes = await getWithRetry(
+          this.client,
+          `/groups/${currentGroupId}/subgroups`,
+          { page: subgroupPage, per_page: 100 }
+        );
+
+        requestCount++;
+        const subgroups = subgroupRes.data || [];
+
+        if (subgroups.length > 0) {
+          for (const sg of subgroups) {
+            if (!visited.has(sg.id)) {
+              toVisit.push(sg.id);
+            }
+          }
+        }
+
+        hasMoreSubgroups = subgroups.length === 100;
+        subgroupPage++;
+      }
     }
-    
+
+    console.log(`[DEBUG] Finished BFS project listing. Total projects=${projects.length}, total requests=${requestCount}`);
     return projects;
   }
-   
+
   async getGroup(groupId) {
     const response = await this.client.get(`/groups/${groupId}`);
     return response.data;
@@ -116,6 +185,28 @@ class GitLabClient {
       throw new Error(`Bulk import API call failed: ${error.response?.status} ${error.response?.statusText} - ${JSON.stringify(error.response?.data)}`);
     }
   }
+
+    async getBulkImportEntitiesAll(importId, { perPage = 100, maxPages = 200 } = {}) {
+      const all = [];
+      let page = 1;
+
+      while (page <= maxPages) {
+        const resp = await getWithRetry(
+          this.client,
+          `/bulk_imports/${importId}/entities`,
+          { page, per_page: perPage }
+        );
+
+        all.push(...(resp.data || []));
+
+        const nextPage = Number(resp.headers?.['x-next-page'] || 0);
+        if (!nextPage) break;
+
+        page = nextPage;
+      }
+
+      return all;
+  }
 }
 
 async function promptUser(name) {
@@ -126,7 +217,7 @@ async function promptUser(name) {
 
   const answer = await rl.question(`Your new group name is ${name}. Are you sure? (Yes/No)`);
 
-  rl.close(); 
+  rl.close();
 
   if (answer.toLowerCase() === 'yes' || answer.toLowerCase() === 'y') {
     console.log("Proceeding...");
@@ -144,6 +235,132 @@ function validateAndConvertRegion(region) {
   return `https://${region}.git.cloud.ibm.com/`;
 }
 
+//  Build a mapping of: old http_url_to_repo -> new http_url_to_repo
+async function generateUrlMappingFile({sourceUrl, destUrl, sourceGroup, destinationGroupPath, sourceProjects}) {
+  const destBase = destUrl.endsWith('/') ? destUrl.slice(0, -1) : destUrl;
+  const urlMapping = {};
+
+  const groupPrefix = `${sourceGroup.full_path}/`;
+
+  for (const project of sourceProjects) {
+    const oldRepoUrl = project.http_url_to_repo; // ends with .git
+
+    // path_with_namespace is like "group/subgroup/project-1"
+    let relativePath;
+    if (project.path_with_namespace.startsWith(groupPrefix)) {
+      relativePath = project.path_with_namespace.slice(groupPrefix.length);
+    } else {
+      // Fallback if for some reason full_path is not a prefix
+      relativePath = project.path_with_namespace;
+    }
+
+    const newRepoUrl = `${destBase}/${destinationGroupPath}/${relativePath}.git`;
+    urlMapping[oldRepoUrl] = newRepoUrl;
+  }
+
+  const mappingFile = 'grit-url-map.json';
+
+  await writeFile(mappingFile, JSON.stringify(urlMapping, null, 2), {
+    encoding: 'utf8',
+  });
+
+  console.log(`\nURL mapping JSON generated at: ${mappingFile}`);
+  console.log(`Total mapped projects: ${sourceProjects.length}`);
+}
+
+function buildGroupImportHistoryUrl(destUrl) {
+  try {
+    return new URL('import/bulk_imports/history', destUrl).toString();
+  } catch {
+    return null;
+  }
+}
+
+function summarizeBulkImportProgress(entities = []) {
+  let entityTotal = 0;
+  let entityFinished = 0;
+  let entityFailed = 0;
+
+  let projectTotal = 0;
+  let projectFinished = 0;
+  let projectFailed = 0;
+
+  let lastCompleted = null;
+  let lastCompletedTs = 0;
+
+  for (const e of entities) {
+    entityTotal++;
+
+    const status = e.status;
+    const isFinished = status === 'finished';
+    const isFailed = status === 'failed';
+
+    if (isFinished) entityFinished++;
+    if (isFailed) entityFailed++;
+
+    const isProjectEntity =
+      e.source_type === 'project_entity' ||
+      e.entity_type === 'project_entity' ||
+      e.entity_type === 'project';
+
+    if (isProjectEntity) {
+      projectTotal++;
+      if (isFinished) projectFinished++;
+      if (isFailed) projectFailed++;
+    }
+
+    if (isFinished) {
+      const ts = new Date(e.updated_at || e.created_at || 0).getTime();
+      if (ts > lastCompletedTs) {
+        lastCompletedTs = ts;
+        lastCompleted = e;
+      }
+    }
+  }
+
+  const entityDone = entityFinished + entityFailed;
+  const entityPct = entityTotal ? Math.floor((entityDone / entityTotal) * 100) : 0;
+
+  const projectDone = projectFinished + projectFailed;
+  const projectPct = projectTotal ? Math.floor((projectDone / projectTotal) * 100) : 0;
+
+  const lastCompletedLabel = lastCompleted?.source_full_path || '';
+
+  return {
+    entityTotal,
+    entityDone,
+    entityFailed,
+    entityPct,
+    projectTotal,
+    projectDone,
+    projectFailed,
+    projectPct,
+    lastCompletedLabel,
+  };
+}
+
+function formatBulkImportProgressLine(importStatus, summary) {
+  if (!summary || summary.entityTotal === 0) {
+    return `Import status: ${importStatus} | Progress: initializing...`;
+  }
+
+  const parts = [`Import status: ${importStatus}`];
+
+  if (summary.projectTotal > 0) {
+    parts.push(`Projects: ${summary.projectDone}/${summary.projectTotal} (${summary.projectPct}%)`);
+    if (summary.projectFailed > 0) parts.push(`Project failed: ${summary.projectFailed}`);
+  }
+
+  parts.push(`Entities: ${summary.entityDone}/${summary.entityTotal} (${summary.entityPct}%)`);
+  if (summary.entityFailed > 0) parts.push(`Failed: ${summary.entityFailed}`);
+
+  if (summary.lastCompletedLabel) {
+    parts.push(`Last completed: ${summary.lastCompletedLabel}`);
+  }
+
+  return parts.join(' | ');
+}
+
 async function directTransfer(options) {
   const sourceUrl = validateAndConvertRegion(options.sourceRegion);
   const destUrl = validateAndConvertRegion(options.destRegion);
@@ -168,6 +385,15 @@ async function directTransfer(options) {
       await promptUser(options.newName);
     }
 
+    // Generate URL mapping JSON before starting the migration
+    await generateUrlMappingFile({
+      sourceUrl,
+      destUrl,
+      sourceGroup,
+      destinationGroupPath,
+      sourceProjects,
+    });
+
     let bulkImport = null;
 
     const requestPayload = {
@@ -181,10 +407,10 @@ async function directTransfer(options) {
         destination_slug: destinationGroupPath,
         destination_namespace: ""
       }]
-    }
+    };
 
     let importRes = null;
-    
+
     try {
       importRes = await destination.bulkImport(requestPayload);
       if (importRes.success) {
@@ -192,28 +418,45 @@ async function directTransfer(options) {
         console.log(`Bulk import request succeeded!`);
         console.log(`Bulk import initiated successfully (ID: ${importRes.data?.id})`);
       } else if (importRes.conflict) {
-      console.log(`Conflict detected: ${importRes.error}`);
-      console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
-      process.exit(0);
+        console.log(`Conflict detected: ${importRes.error}`);
+        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
+        process.exit(0);
       }
     } catch (error) {
       console.log(`Bulk import request failed - ${error.message}`);
       process.exit(0);
     }
 
-    console.log('\nPolling bulk import status (checking every 5 minute)...');
+    console.log('\nPolling bulk import status (adaptive: 1m→2m→3m→4m→5m, max 60 checks)...');
+    const MAX_ATTEMPTS = 60;
+    const POLLS_PER_STEP = 5;
+    const MIN_INTERVAL_MIN = 1;
+    const MAX_INTERVAL_MIN = 5;
+
     let importStatus = 'created';
     let attempts = 0;
-    
-    while (!['finished', 'failed', 'timeout'].includes(importStatus) && attempts < 60) {
+
+    while (!['finished', 'failed', 'timeout'].includes(importStatus) && attempts < MAX_ATTEMPTS) {
       if (attempts > 0) {
-        console.log(`Waiting 5 minute before next status check...`);
-        await new Promise(resolve => setTimeout(resolve, 5*60000));
+        const step = Math.floor(attempts / POLLS_PER_STEP);
+        const waitMin = Math.min(MIN_INTERVAL_MIN + step, MAX_INTERVAL_MIN);
+
+        console.log(`Waiting ${waitMin} minute before next status check...`);
+        await new Promise(resolve => setTimeout(resolve, waitMin * 60000));
       }
       try {
         const importDetails = await destination.getBulkImport(bulkImport.id);
         importStatus = importDetails.status;
-        console.log(`[${new Date().toLocaleTimeString()}] Import status: ${importStatus}`);
+        let progressLine;
+        try {
+          const entitiesAll = await destination.getBulkImportEntitiesAll(bulkImport.id);
+          const summary = summarizeBulkImportProgress(entitiesAll);
+          progressLine = formatBulkImportProgressLine(importStatus, summary);
+        } catch {
+          progressLine = `Import status: ${importStatus} | Progress: (unable to fetch entity details)`;
+        }
+
+        console.log(`[${new Date().toLocaleTimeString()}] ${progressLine}`);
 
         if (importStatus === 'finished') {
           console.log('Bulk import completed successfully!');
@@ -230,29 +473,40 @@ async function directTransfer(options) {
       }
       attempts++;
     }
-    
-    if (attempts >= 60) {
-      console.error(`Bulk import either timed out or is still running in the background`);
+
+    if (attempts >= MAX_ATTEMPTS) {
+      const historyUrl = buildGroupImportHistoryUrl(destUrl);
+
+      console.error('\nThe CLI has stopped polling for the GitLab bulk import.');
+      console.error('The migration itself may still be running inside GitLab — the CLI only waits for a limited time.');
+      console.error(`Last reported status for bulk import ${bulkImport.id}: ${importStatus}`);
+
+      if (historyUrl) {
+        console.error('\nYou can continue monitoring this migration in the GitLab UI.');
+        console.error(`Group import history: ${historyUrl}`);
+      } else {
+        console.error('\nYou can continue monitoring this migration from the Group import history page in the GitLab UI.');
+      }
       process.exit(0);
     }
 
     const entities = await destination.getBulkImportEntities(bulkImport.id);
     const finishedEntities = entities.filter(e => e.status === 'finished');
     const failedEntities = entities.filter(e => e.status === 'failed');
-    
+
     if (importStatus === 'finished' && finishedEntities.length > 0) {
       console.log(`\nGroup migration completed successfully!`);
       console.log(`Migration Results:`);
       console.log(`Successfully migrated: ${finishedEntities.length} entities`);
       console.log(`Failed: ${failedEntities.length} entities`);
-      
+
       if (failedEntities.length > 0) {
         console.log(`\nFailed entities:\n`);
         failedEntities.forEach(e => {
           console.log(`${e.source_type}: ${e.source_full_path} (${e.status})`);
         });
       }
-      
+
       return 0;
     } else {
       console.error('\nBulk import failed!');
@@ -282,7 +536,7 @@ const command = new Command('copy-project-group')
   .showHelpAfterError()
   .hook('preAction', cmd => cmd.showHelpAfterError(false)) // only show help during validation
   .action(async (options) => {
-    await directTransfer(options);  
+    await directTransfer(options);
   });
 
 export default command;

package/cmd/utils/requests.js

@@ -477,6 +477,28 @@ async function migrateToolchainSecrets(bearer, data, region) {
     }
 }
 
+// GET with retry for flaky 5xx/520 errors (Cloudflare / origin issues)
+async function getWithRetry(client, path, params = {}, { retries = 3, retryDelayMs = 2000 } = {}) {
+  let lastError;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await client.get(path, { params });
+    } catch (error) {
+      const status = error.response?.status;
+      if (attempt < retries && status && status >= 500) {
+        console.warn(
+          `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
+        );
+        await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
+        lastError = error;
+        continue;
+      }
+      throw error; // Non-5xx or out of retries: rethrow
+    }
+  }
+  throw lastError;
+}
+
 export {
     getBearerToken,
     getAccountId,
@@ -495,5 +517,6 @@ export {
     deleteToolchain,
     createTool,
     getSmInstances,
-    migrateToolchainSecrets
+    migrateToolchainSecrets,
+    getWithRetry
 }

package/cmd/utils/terraform.js

@@ -205,7 +205,7 @@ async function setupTerraformFiles({ token, srcRegion, targetRegion, targetTag,
                                 logger.print('Please enter the new URLs for the following GRIT tool(s) (or submit empty input to skip):\n');
                             }
 
-                            const newRepoSlug = await promptUserInput(`Old URL: ${thisUrl.slice(0, thisUrl.length - 4)}\nNew URL: ${GIT_BASE_URL || 'https://' + targetRegion + '.git.cloud.ibm.com'}`, '', validateGritUrlPrompt);
+                            const newRepoSlug = await promptUserInput(`Old URL: ${thisUrl.slice(0, thisUrl.length - 4)}\nNew URL: ${GIT_BASE_URL || 'https://' + targetRegion + '.git.cloud.ibm.com'}/`, '', validateGritUrlPrompt);
 
                             if (newRepoSlug) {
                                 newUrl = (GIT_BASE_URL || `https://${targetRegion}.git.cloud.ibm.com`) + `/${newRepoSlug}.git`;

package/cmd/utils/validate.js

@@ -405,14 +405,17 @@ async function validateOAuth(token, tools, targetRegion, skipPrompt) {
 
 async function validateGritUrl(token, region, url, validateFull) {
     if (typeof url != 'string') throw Error('Provided GRIT url is not a string');
-    let trimmed;
+    let trimmed = url.trim();
 
     if (validateFull) {
         const baseUrl = (GIT_BASE_URL || `https://${region}.git.cloud.ibm.com`) + '/';
-        if (!url.startsWith(baseUrl) || !url.endsWith('.git')) throw Error('Provided full GRIT url is not valid');
-        trimmed = url.slice(baseUrl.length, url.length - '.git'.length);
-    } else {
-        trimmed = url.trim();
+        if (!trimmed.startsWith(baseUrl)) throw Error('Provided full GRIT url is not valid');
+
+        if (trimmed.endsWith('.git')) {
+            trimmed = trimmed.slice(baseUrl.length, trimmed.length - '.git'.length);
+        } else {
+            trimmed = trimmed.slice(baseUrl.length);
+        }
     }
 
     // split into two parts, user/group/subgroup and project
@@ -468,7 +471,7 @@ async function validateGritUrl(token, region, url, validateFull) {
         await getGritGroupProject(accessToken, region, urlStart, projectName);
         return trimmed;
     } catch {
-        throw Error('Provided GRIT url not found');
+        throw Error(`Provided GRIT url not found: ${url}`);
     }
 }
 

package/config.js

@@ -11,9 +11,9 @@ const COPY_TOOLCHAIN_DESC = `Copies a toolchain, including tool integrations and
 
 Examples:
   export IBMCLOUD_API_KEY='...'
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r us-south
+  npx @ibm-cloud/cd-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r us-south
       Copy a toolchain to the Dallas region with the same name, in the same resource group.
-  npx @ibm-cloud/cd-migration-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey \${APIKEY}
+  npx @ibm-cloud/cd-tools copy-toolchain -c \${TOOLCHAIN_CRN} -r eu-de -n new-toolchain-name -g new-resource-group --apikey \${APIKEY}
       Copy a toolchain to the Frankfurt region with the specified name and target resource group, using the given API key
 
 Environment Variables:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.6.1",
+  "version": "1.8.1",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",