@ibm-cloud/cd-tools 1.8.1 → 1.9.0

package/cmd/copy-toolchain.js

@@ -324,6 +324,9 @@ async function main(options) {
 			// create toolchain, which invokes script to create s2s if applicable
 			await runTerraformApply(true, outputDir, verbosity, `ibm_cd_toolchain.${toolchainTfName}`);
 
+			const hasS2SFailures = fs.existsSync(resolve(`${outputDir}/.s2s-script-failures`));
+			if (hasS2SFailures) logger.warn('\nWarning! One or more service-to-service auth policies could not be created!\n');
+
 			// create the rest
 			await runTerraformApply(skipUserConfirmation, outputDir, verbosity).catch((err) => {
 				logger.error(err, LOG_STAGES.tf);
@@ -335,6 +338,7 @@ async function main(options) {
 
 			logger.print('\n');
 			logger.info(`Toolchain "${sourceToolchainData['name']}" from ${sourceRegion} was cloned to "${targetToolchainName ?? sourceToolchainData['name']}" in ${targetRegion} ${applyErrors ? 'with some errors' : 'successfully'}, with ${numResourcesCreated} / ${numResourcesPlanned} resources created!`, LOG_STAGES.info);
+			if (hasS2SFailures) logger.warn('One or more service-to-service auth policies could not be created, see .s2s-script-failures for more details.');
 			if (newTcId) logger.info(`See cloned toolchain: https://${CLOUD_PLATFORM}/devops/toolchains/${newTcId}?env_id=ibm:yp:${targetRegion}`, LOG_STAGES.info, true);
 		} else {
 			logger.info(`DRY_RUN: ${dryRun}, skipping terraform apply...`, LOG_STAGES.tf);

package/cmd/direct-transfer.js

@@ -207,6 +207,20 @@ class GitLabClient {
 
       return all;
   }
+
+  async getGroupByFullPath(fullPath) {
+    const encoded = encodeURIComponent(fullPath);
+    const resp = await this.client.get(`/groups/${encoded}`);
+    return resp.data;
+  }
+
+  async listBulkImports({ page = 1, perPage = 50 } = {}) {
+    const resp = await getWithRetry(this.client, `/bulk_imports`, { page, per_page: perPage });
+    return {
+      imports: resp.data || [],
+      nextPage: Number(resp.headers?.['x-next-page'] || 0),
+    };
+  }
 }
 
 async function promptUser(name) {
@@ -361,6 +375,84 @@ function formatBulkImportProgressLine(importStatus, summary) {
   return parts.join(' | ');
 }
 
+function buildGroupUrl(base, path) {
+  try {
+    return new URL(path.replace(/^\//, ''), base).toString();
+  } catch {
+    return null;
+  }
+}
+
+function isGroupEntity(e) {
+  return e?.source_type === 'group_entity' || e?.entity_type === 'group_entity' || e?.entity_type === 'group';
+}
+
+async function handleBulkImportConflict({destination, destUrl, sourceGroupFullPath, destinationGroupPath, importResErr}) {
+  const historyUrl = buildGroupImportHistoryUrl(destUrl);
+  const groupUrl = buildGroupUrl(destUrl, `/groups/${destinationGroupPath}`);
+  const fallback = () => {
+    console.log(`\nDestination group already exists.`);
+    if (groupUrl) console.log(`Group: ${groupUrl}`);
+    if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+    process.exit(0);
+  };
+
+  try {
+    await destination.getGroupByFullPath(destinationGroupPath);
+  } catch {
+    fallback();
+  }
+
+  try {
+    const IMPORT_PAGES = 3;
+    const ENTITY_PAGES = 2;
+
+    let page = 1;
+    for (let p = 0; p < IMPORT_PAGES; p++) {
+      const { imports, nextPage } = await destination.listBulkImports({ page, perPage: 50 });
+
+      for (const bi of imports) {
+        if (!bi?.id) continue;
+
+        const status = bi.status;
+        if (!['created', 'started', 'finished'].includes(status)) continue;
+
+        const entities = await destination.getBulkImportEntitiesAll(bi.id, { perPage: 100, maxPages: ENTITY_PAGES });
+
+        const matchesThisGroup = entities.some(e =>
+          isGroupEntity(e) &&
+          e.source_full_path === sourceGroupFullPath &&
+          (e.destination_full_path === destinationGroupPath || e.destination_slug === destinationGroupPath)
+        );
+
+        if (!matchesThisGroup) continue;
+
+        if (status === 'created' || status === 'started') {
+          console.log(`\nGroup is already in migration...`);
+          console.log(`Bulk import ID: ${bi.id}`);
+          if (groupUrl) console.log(`Migrated group: ${groupUrl}`);
+          if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+          process.exit(0);
+        }
+
+        console.log(`\nConflict detected: ${importResErr}`);
+        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
+        console.log(`\nGroup already migrated.`);
+        if (groupUrl) console.log(`Migrated group: ${groupUrl}`);
+        if (historyUrl) console.log(`Group import history: ${historyUrl}`);
+        process.exit(0);
+      }
+
+      if (!nextPage) break;
+      page = nextPage;
+    }
+
+    fallback();
+  } catch {
+    fallback();
+  }
+}
+
 async function directTransfer(options) {
   const sourceUrl = validateAndConvertRegion(options.sourceRegion);
   const destUrl = validateAndConvertRegion(options.destRegion);
@@ -418,9 +510,13 @@ async function directTransfer(options) {
         console.log(`Bulk import request succeeded!`);
         console.log(`Bulk import initiated successfully (ID: ${importRes.data?.id})`);
       } else if (importRes.conflict) {
-        console.log(`Conflict detected: ${importRes.error}`);
-        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
-        process.exit(0);
+        await handleBulkImportConflict({
+          destination,
+          destUrl,
+          sourceGroupFullPath: sourceGroup.full_path,
+          destinationGroupPath,
+          importResErr: importRes.error
+        });
       }
     } catch (error) {
       console.log(`Bulk import request failed - ${error.message}`);
@@ -506,6 +602,8 @@ async function directTransfer(options) {
           console.log(`${e.source_type}: ${e.source_full_path} (${e.status})`);
         });
       }
+      const migratedGroupUrl = buildGroupUrl(destUrl, `/groups/${destinationGroupPath}`);
+      if (migratedGroupUrl) console.log(`\nMigrated group: ${migratedGroupUrl}`);
 
       return 0;
     } else {

package/cmd/utils/terraform.js

@@ -492,7 +492,7 @@ function replaceDependsOn(str) {
 function addS2sScriptToToolchainTf(str) {
     const provisionerStr = (tfName) => `\n\n  provisioner "local-exec" {
     command = "node create-s2s-script.cjs"
-    on_failure = fail
+    on_failure = continue
     environment = {
       IBMCLOUD_API_KEY = var.ibmcloud_api_key
       TARGET_TOOLCHAIN_ID = ibm_cd_toolchain.${tfName}.id

package/create-s2s-script.js

@@ -23,7 +23,8 @@ if (!CLOUD_PLATFORM) throw Error(`Missing 'IBMCLOUD_PLATFORM'`);
 const IAM_BASE_URL = process.env['IAM_BASE_URL'] || 'https://iam.cloud.ibm.com';
 if (!IAM_BASE_URL) throw Error(`Missing 'IAM_BASE_URL'`);
 
-const INPUT_PATH = 'create-s2s.json';
+const INPUT_PATH = resolve('create-s2s.json');
+const ERROR_PATH = resolve('.s2s-script-failures');
 
 async function getBearer() {
     const url = `${IAM_BASE_URL}/identity/token`;
@@ -44,14 +45,12 @@ async function getBearer() {
         });
 
         if (!response.ok) {
-            throw new Error(`Response status: ${response.status}, ${response.statusText}`);
+            throw new Error(`Failed to get bearer token with status: ${response.status}, ${response.statusText}`);
         }
 
-        console.log(`GETTING BEARER TOKEN... ${response.status}, ${response.statusText}`);
-
         return (await response.json()).access_token;
     } catch (error) {
-        console.error(error.message);
+        console.error(`Failed to get bearer token: ${error.message}`);
     }
 }
 
@@ -106,17 +105,20 @@ async function createS2sAuthPolicy(bearer, item) {
         if (!response.ok) {
             return Promise.reject(`Failed to create service-to-service authorization policy for ${item['serviceId']} '${item['parameters']['label'] ?? item['parameters']['name']}' with status: ${response.status} ${response.statusText}`);
         }
-
-        console.log(`CREATING AUTH POLICY... ${response.status}, ${response.statusText}`);
     } catch (error) {
-        return Promise.reject(error.message);
+        return Promise.reject(`Failed to create service-to-service authorization policy for ${item['serviceId']} '${error.message}`);
     }
 }
 
 // main
 
 getBearer().then(async (bearer) => {
-    const inputArr = JSON.parse(fs.readFileSync(resolve(INPUT_PATH)));
+    // remove temp file from previous runs
+    if (fs.existsSync(ERROR_PATH)) {
+        fs.rmSync(ERROR_PATH);
+    }
+
+    const inputArr = JSON.parse(fs.readFileSync(INPUT_PATH));
 
     const promises = [];
     inputArr.forEach((item) => {
@@ -126,7 +128,9 @@ getBearer().then(async (bearer) => {
     try {
         await Promise.all(promises);
     } catch (e) {
-        console.error(e)
+        console.error(e);
+        // create temp file on error
+        fs.writeFileSync(ERROR_PATH, e);
         exit(1);
     }
 });

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.8.1",
+  "version": "1.9.0",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",