@ibm-cloud/cd-tools 1.7.0 → 1.8.2

package/cmd/direct-transfer.js

@@ -12,6 +12,7 @@ import axios from 'axios';
 import readline from 'readline/promises';
 import { writeFile } from 'fs/promises';
 import { TARGET_REGIONS, SOURCE_REGIONS } from '../config.js';
+import { getWithRetry } from './utils/requests.js';
 
 const HTTP_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes default
 
@@ -55,7 +56,8 @@ class GitLabClient {
           return projects;
         }
 
-        const projRes = await this.getWithRetry(
+        const projRes = await getWithRetry(
+          this.client,
           `/groups/${currentGroupId}/projects`,
           { page: projPage, per_page: 100 }
         );
@@ -82,7 +84,8 @@ class GitLabClient {
           return projects;
         }
 
-        const subgroupRes = await this.getWithRetry(
+        const subgroupRes = await getWithRetry(
+          this.client,
           `/groups/${currentGroupId}/subgroups`,
           { page: subgroupPage, per_page: 100 }
         );
@@ -107,30 +110,6 @@ class GitLabClient {
     return projects;
   }
 
-  // Helper: GET with retry for flaky 5xx/520 errors (Cloudflare / origin issues)
-  async getWithRetry(path, params = {}, { retries = 3, retryDelayMs = 2000 } = {}) {
-    let lastError;
-    for (let attempt = 1; attempt <= retries; attempt++) {
-      try {
-        return await this.client.get(path, { params });
-      } catch (error) {
-        const status = error.response?.status;
-
-        if (attempt < retries && status && status >= 500) {
-          console.warn(
-            `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
-          );
-          await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
-          lastError = error;
-          continue;
-        }
-
-        throw error; // Non-5xx or out of retries: rethrow
-      }
-    }
-    throw lastError;
-  }
-
   async getGroup(groupId) {
     const response = await this.client.get(`/groups/${groupId}`);
     return response.data;
@@ -206,6 +185,42 @@ class GitLabClient {
       throw new Error(`Bulk import API call failed: ${error.response?.status} ${error.response?.statusText} - ${JSON.stringify(error.response?.data)}`);
     }
   }
+
+    async getBulkImportEntitiesAll(importId, { perPage = 100, maxPages = 200 } = {}) {
+      const all = [];
+      let page = 1;
+
+      while (page <= maxPages) {
+        const resp = await getWithRetry(
+          this.client,
+          `/bulk_imports/${importId}/entities`,
+          { page, per_page: perPage }
+        );
+
+        all.push(...(resp.data || []));
+
+        const nextPage = Number(resp.headers?.['x-next-page'] || 0);
+        if (!nextPage) break;
+
+        page = nextPage;
+      }
+
+      return all;
+  }
+
+  async getGroupByFullPath(fullPath) {
+    const encoded = encodeURIComponent(fullPath);
+    const resp = await this.client.get(`/groups/${encoded}`);
+    return resp.data;
+  }
+
+  async listBulkImports({ page = 1, perPage = 50 } = {}) {
+    const resp = await getWithRetry(this.client, `/bulk_imports`, { page, per_page: perPage });
+    return {
+      imports: resp.data || [],
+      nextPage: Number(resp.headers?.['x-next-page'] || 0),
+    };
+  }
 }
 
 async function promptUser(name) {
@@ -234,7 +249,7 @@ function validateAndConvertRegion(region) {
   return `https://${region}.git.cloud.ibm.com/`;
 }
 
-//  Build a mapping of: old http_url_to_repo -> new http_url_to_repo and old web_url -> new web_url
+//  Build a mapping of: old http_url_to_repo -> new http_url_to_repo
 async function generateUrlMappingFile({sourceUrl, destUrl, sourceGroup, destinationGroupPath, sourceProjects}) {
   const destBase = destUrl.endsWith('/') ? destUrl.slice(0, -1) : destUrl;
   const urlMapping = {};
@@ -267,6 +282,177 @@ async function generateUrlMappingFile({sourceUrl, destUrl, sourceGroup, destinat
   console.log(`Total mapped projects: ${sourceProjects.length}`);
 }
 
+function buildGroupImportHistoryUrl(destUrl) {
+  try {
+    return new URL('import/bulk_imports/history', destUrl).toString();
+  } catch {
+    return null;
+  }
+}
+
+function summarizeBulkImportProgress(entities = []) {
+  let entityTotal = 0;
+  let entityFinished = 0;
+  let entityFailed = 0;
+
+  let projectTotal = 0;
+  let projectFinished = 0;
+  let projectFailed = 0;
+
+  let lastCompleted = null;
+  let lastCompletedTs = 0;
+
+  for (const e of entities) {
+    entityTotal++;
+
+    const status = e.status;
+    const isFinished = status === 'finished';
+    const isFailed = status === 'failed';
+
+    if (isFinished) entityFinished++;
+    if (isFailed) entityFailed++;
+
+    const isProjectEntity =
+      e.source_type === 'project_entity' ||
+      e.entity_type === 'project_entity' ||
+      e.entity_type === 'project';
+
+    if (isProjectEntity) {
+      projectTotal++;
+      if (isFinished) projectFinished++;
+      if (isFailed) projectFailed++;
+    }
+
+    if (isFinished) {
+      const ts = new Date(e.updated_at || e.created_at || 0).getTime();
+      if (ts > lastCompletedTs) {
+        lastCompletedTs = ts;
+        lastCompleted = e;
+      }
+    }
+  }
+
+  const entityDone = entityFinished + entityFailed;
+  const entityPct = entityTotal ? Math.floor((entityDone / entityTotal) * 100) : 0;
+
+  const projectDone = projectFinished + projectFailed;
+  const projectPct = projectTotal ? Math.floor((projectDone / projectTotal) * 100) : 0;
+
+  const lastCompletedLabel = lastCompleted?.source_full_path || '';
+
+  return {
+    entityTotal,
+    entityDone,
+    entityFailed,
+    entityPct,
+    projectTotal,
+    projectDone,
+    projectFailed,
+    projectPct,
+    lastCompletedLabel,
+  };
+}
+
+function formatBulkImportProgressLine(importStatus, summary) {
+  if (!summary || summary.entityTotal === 0) {
+    return `Import status: ${importStatus} | Progress: initializing...`;
+  }
+
+  const parts = [`Import status: ${importStatus}`];
+
+  if (summary.projectTotal > 0) {
+    parts.push(`Projects: ${summary.projectDone}/${summary.projectTotal} (${summary.projectPct}%)`);
+    if (summary.projectFailed > 0) parts.push(`Project failed: ${summary.projectFailed}`);
+  }
+
+  parts.push(`Entities: ${summary.entityDone}/${summary.entityTotal} (${summary.entityPct}%)`);
+  if (summary.entityFailed > 0) parts.push(`Failed: ${summary.entityFailed}`);
+
+  if (summary.lastCompletedLabel) {
+    parts.push(`Last completed: ${summary.lastCompletedLabel}`);
+  }
+
+  return parts.join(' | ');
+}
+
+function buildGroupUrl(base, path) {
+  try {
+    return new URL(path.replace(/^\//, ''), base).toString();
+  } catch {
+    return null;
+  }
+}
+
+function isGroupEntity(e) {
+  return e?.source_type === 'group_entity' || e?.entity_type === 'group_entity' || e?.entity_type === 'group';
+}
+
+async function handleBulkImportConflict({destination, destUrl, sourceGroupFullPath, destinationGroupPath, importResErr}) {
+  const historyUrl = buildGroupImportHistoryUrl(destUrl);
+  const groupUrl = buildGroupUrl(destUrl, `/groups/${destinationGroupPath}`);
+  const fallback = () => {
+    console.log(`\nDestination group already exists.`);
+    if (groupUrl) console.log(`Group: ${groupUrl}`);
+    if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+    process.exit(0);
+  };
+
+  try {
+    await destination.getGroupByFullPath(destinationGroupPath);
+  } catch {
+    fallback();
+  }
+
+  try {
+    const IMPORT_PAGES = 3;
+    const ENTITY_PAGES = 2;
+
+    let page = 1;
+    for (let p = 0; p < IMPORT_PAGES; p++) {
+      const { imports, nextPage } = await destination.listBulkImports({ page, perPage: 50 });
+
+      for (const bi of imports) {
+        if (!bi?.id) continue;
+
+        const status = bi.status;
+        if (!['created', 'started', 'finished'].includes(status)) continue;
+
+        const entities = await destination.getBulkImportEntitiesAll(bi.id, { perPage: 100, maxPages: ENTITY_PAGES });
+
+        const matchesThisGroup = entities.some(e =>
+          isGroupEntity(e) &&
+          e.source_full_path === sourceGroupFullPath &&
+          (e.destination_full_path === destinationGroupPath || e.destination_slug === destinationGroupPath)
+        );
+
+        if (!matchesThisGroup) continue;
+
+        if (status === 'created' || status === 'started') {
+          console.log(`\nGroup is already in migration...`);
+          console.log(`Bulk import ID: ${bi.id}`);
+          if (groupUrl) console.log(`Migrated group: ${groupUrl}`);
+          if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+          process.exit(0);
+        }
+
+        console.log(`\nConflict detected: ${importResErr}`);
+        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
+        console.log(`\nGroup already migrated.`);
+        if (groupUrl) console.log(`Migrated group: ${groupUrl}`);
+        if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+        process.exit(0);
+      }
+
+      if (!nextPage) break;
+      page = nextPage;
+    }
+
+    fallback();
+  } catch {
+    fallback();
+  }
+}
+
 async function directTransfer(options) {
   const sourceUrl = validateAndConvertRegion(options.sourceRegion);
   const destUrl = validateAndConvertRegion(options.destRegion);
@@ -324,28 +510,49 @@ async function directTransfer(options) {
         console.log(`Bulk import request succeeded!`);
         console.log(`Bulk import initiated successfully (ID: ${importRes.data?.id})`);
       } else if (importRes.conflict) {
-        console.log(`Conflict detected: ${importRes.error}`);
-        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
-        process.exit(0);
+        await handleBulkImportConflict({
+          destination,
+          destUrl,
+          sourceGroupFullPath: sourceGroup.full_path,
+          destinationGroupPath,
+          importResErr: importRes.error
+        });
       }
     } catch (error) {
       console.log(`Bulk import request failed - ${error.message}`);
       process.exit(0);
     }
 
-    console.log('\nPolling bulk import status (checking every 5 minute)...');
+    console.log('\nPolling bulk import status (adaptive: 1m→2m→3m→4m→5m, max 60 checks)...');
+    const MAX_ATTEMPTS = 60;
+    const POLLS_PER_STEP = 5;
+    const MIN_INTERVAL_MIN = 1;
+    const MAX_INTERVAL_MIN = 5;
+
     let importStatus = 'created';
     let attempts = 0;
 
-    while (!['finished', 'failed', 'timeout'].includes(importStatus) && attempts < 60) {
+    while (!['finished', 'failed', 'timeout'].includes(importStatus) && attempts < MAX_ATTEMPTS) {
       if (attempts > 0) {
-        console.log(`Waiting 5 minute before next status check...`);
-        await new Promise(resolve => setTimeout(resolve, 5 * 60000));
+        const step = Math.floor(attempts / POLLS_PER_STEP);
+        const waitMin = Math.min(MIN_INTERVAL_MIN + step, MAX_INTERVAL_MIN);
+
+        console.log(`Waiting ${waitMin} minute before next status check...`);
+        await new Promise(resolve => setTimeout(resolve, waitMin * 60000));
       }
       try {
         const importDetails = await destination.getBulkImport(bulkImport.id);
         importStatus = importDetails.status;
-        console.log(`[${new Date().toLocaleTimeString()}] Import status: ${importStatus}`);
+        let progressLine;
+        try {
+          const entitiesAll = await destination.getBulkImportEntitiesAll(bulkImport.id);
+          const summary = summarizeBulkImportProgress(entitiesAll);
+          progressLine = formatBulkImportProgressLine(importStatus, summary);
+        } catch {
+          progressLine = `Import status: ${importStatus} | Progress: (unable to fetch entity details)`;
+        }
+
+        console.log(`[${new Date().toLocaleTimeString()}] ${progressLine}`);
 
         if (importStatus === 'finished') {
           console.log('Bulk import completed successfully!');
@@ -363,8 +570,19 @@ async function directTransfer(options) {
       attempts++;
     }
 
-    if (attempts >= 60) {
-      console.error(`Bulk import either timed out or is still running in the background`);
+    if (attempts >= MAX_ATTEMPTS) {
+      const historyUrl = buildGroupImportHistoryUrl(destUrl);
+
+      console.error('\nThe CLI has stopped polling for the GitLab bulk import.');
+      console.error('The migration itself may still be running inside GitLab — the CLI only waits for a limited time.');
+      console.error(`Last reported status for bulk import ${bulkImport.id}: ${importStatus}`);
+
+      if (historyUrl) {
+        console.error('\nYou can continue monitoring this migration in the GitLab UI.');
+        console.error(`Group import history: ${historyUrl}`);
+      } else {
+        console.error('\nYou can continue monitoring this migration from the Group import history page in the GitLab UI.');
+      }
       process.exit(0);
     }
 
@@ -384,6 +602,8 @@ async function directTransfer(options) {
           console.log(`${e.source_type}: ${e.source_full_path} (${e.status})`);
         });
       }
+      const migratedGroupUrl = buildGroupUrl(destUrl, `/groups/${destinationGroupPath}`);
+      if (migratedGroupUrl) console.log(`\nMigrated group: ${migratedGroupUrl}`);
 
       return 0;
     } else {

package/cmd/utils/requests.js

@@ -477,6 +477,28 @@ async function migrateToolchainSecrets(bearer, data, region) {
     }
 }
 
+// GET with retry for flaky 5xx/520 errors (Cloudflare / origin issues)
+async function getWithRetry(client, path, params = {}, { retries = 3, retryDelayMs = 2000 } = {}) {
+  let lastError;
+  for (let attempt = 1; attempt <= retries; attempt++) {
+    try {
+      return await client.get(path, { params });
+    } catch (error) {
+      const status = error.response?.status;
+      if (attempt < retries && status && status >= 500) {
+        console.warn(
+          `[WARN] GET ${path} failed with status ${status} (attempt ${attempt}/${retries}). Retrying...`
+        );
+        await new Promise(resolve => setTimeout(resolve, retryDelayMs * attempt));
+        lastError = error;
+        continue;
+      }
+      throw error; // Non-5xx or out of retries: rethrow
+    }
+  }
+  throw lastError;
+}
+
 export {
     getBearerToken,
     getAccountId,
@@ -495,5 +517,6 @@ export {
     deleteToolchain,
     createTool,
     getSmInstances,
-    migrateToolchainSecrets
+    migrateToolchainSecrets,
+    getWithRetry
 }

package/cmd/utils/terraform.js

@@ -205,7 +205,7 @@ async function setupTerraformFiles({ token, srcRegion, targetRegion, targetTag,
                                 logger.print('Please enter the new URLs for the following GRIT tool(s) (or submit empty input to skip):\n');
                             }
 
-                            const newRepoSlug = await promptUserInput(`Old URL: ${thisUrl.slice(0, thisUrl.length - 4)}\nNew URL: ${GIT_BASE_URL || 'https://' + targetRegion + '.git.cloud.ibm.com'}`, '', validateGritUrlPrompt);
+                            const newRepoSlug = await promptUserInput(`Old URL: ${thisUrl.slice(0, thisUrl.length - 4)}\nNew URL: ${GIT_BASE_URL || 'https://' + targetRegion + '.git.cloud.ibm.com'}/`, '', validateGritUrlPrompt);
 
                             if (newRepoSlug) {
                                 newUrl = (GIT_BASE_URL || `https://${targetRegion}.git.cloud.ibm.com`) + `/${newRepoSlug}.git`;

package/cmd/utils/validate.js

@@ -405,14 +405,17 @@ async function validateOAuth(token, tools, targetRegion, skipPrompt) {
 
 async function validateGritUrl(token, region, url, validateFull) {
     if (typeof url != 'string') throw Error('Provided GRIT url is not a string');
-    let trimmed;
+    let trimmed = url.trim();
 
     if (validateFull) {
         const baseUrl = (GIT_BASE_URL || `https://${region}.git.cloud.ibm.com`) + '/';
-        if (!url.startsWith(baseUrl) || !url.endsWith('.git')) throw Error('Provided full GRIT url is not valid');
-        trimmed = url.slice(baseUrl.length, url.length - '.git'.length);
-    } else {
-        trimmed = url.trim();
+        if (!trimmed.startsWith(baseUrl)) throw Error('Provided full GRIT url is not valid');
+
+        if (trimmed.endsWith('.git')) {
+            trimmed = trimmed.slice(baseUrl.length, trimmed.length - '.git'.length);
+        } else {
+            trimmed = trimmed.slice(baseUrl.length);
+        }
     }
 
     // split into two parts, user/group/subgroup and project
@@ -468,7 +471,7 @@ async function validateGritUrl(token, region, url, validateFull) {
         await getGritGroupProject(accessToken, region, urlStart, projectName);
         return trimmed;
     } catch {
-        throw Error('Provided GRIT url not found');
+        throw Error(`Provided GRIT url not found: ${url}`);
     }
 }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.7.0",
+  "version": "1.8.2",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",