@ibm-cloud/cd-tools 1.2.1 → 1.2.3

package/cmd/check-secrets.js

@@ -49,11 +49,15 @@ async function main(options) {
         for (let i = 0; i < getToolsRes.tools.length; i++) {
             const tool = getToolsRes.tools[i];
 
+            // Skip iff it's GitHub/GitLab/GRIT integration with OAuth
+            if (['githubconsolidated', 'github_integrated', 'gitlab', 'hostedgit'].includes(tool.tool_type_id) && (tool.parameters?.auth_type === '' || tool.parameters?.auth_type === 'oauth'))
+                continue;
+
             // Check tool integrations for any plain text secret values
             if (SECRET_KEYS_MAP[tool.tool_type_id]) {
                 SECRET_KEYS_MAP[tool.tool_type_id].forEach((entry) => {
                     const updateableSecretParam = entry.key;
-                    if (tool.parameters[updateableSecretParam] && !isSecretReference(tool.parameters[updateableSecretParam])) {
+                    if (tool.parameters[updateableSecretParam] && !isSecretReference(tool.parameters[updateableSecretParam]) && tool.parameters[updateableSecretParam].length > 0) {
                         toolResults.push({
                             'Tool ID': tool.id,
                             'Tool Type': tool.tool_type_id,
@@ -68,7 +72,7 @@ async function main(options) {
                 const pipelineData = await getPipelineData(token, tool.id, region);
 
                 pipelineData?.properties.forEach((prop) => {
-                    if (prop.type === 'secure' && !isSecretReference(prop.value)) {
+                    if (prop.type === 'secure' && !isSecretReference(prop.value) && prop.value.length > 0) {
                         pipelineResults.push({
                             'Pipeline ID': pipelineData.id,
                             'Trigger Name': '-',
@@ -79,7 +83,7 @@ async function main(options) {
 
                 pipelineData?.triggers.forEach((trigger) => {
                     trigger.properties?.forEach((prop) => {
-                        if (prop.type === 'secure' && !isSecretReference(prop.value)) {
+                        if (prop.type === 'secure' && !isSecretReference(prop.value) && prop.value.length > 0) {
                             pipelineResults.push({
                                 'Pipeline ID': pipelineData.id,
                                 'Trigger Name': trigger.name,

package/cmd/copy-toolchain.js

@@ -16,7 +16,7 @@ import { Command, Option } from 'commander';
 import { parseEnvVar } from './utils/utils.js';
 import { logger, LOG_STAGES } from './utils/logger.js';
 import { setTerraformEnv, initProviderFile, setupTerraformFiles, runTerraformInit, getNumResourcesPlanned, runTerraformApply, getNumResourcesCreated, getNewToolchainId } from './utils/terraform.js';
-import { getAccountId, getBearerToken, getIamAuthPolicies, getResourceGroupIdAndName, getToolchain } from './utils/requests.js';
+import { getAccountId, getBearerToken, getCdInstanceByRegion, getIamAuthPolicies, getResourceGroupIdAndName, getToolchain } from './utils/requests.js';
 import { validatePrereqsVersions, validateTag, validateToolchainId, validateToolchainName, validateTools, validateOAuth, warnDuplicateName, validateGritUrl } from './utils/validate.js';
 import { importTerraform } from './utils/import-terraform.js';
 
@@ -26,11 +26,12 @@ process.on('exit', (code) => {
 	if (code !== 0) logger.print(`Need help? Visit ${MIGRATION_DOC_URL} for more troubleshooting information.`);
 });
 
+const TIME_SUFFIX = new Date().getTime();
 const LOGS_DIR = '.logs';
-const TEMP_DIR = '.migration-temp'
+const TEMP_DIR = '.migration-temp-' + TIME_SUFFIX;
 const LOG_DUMP = process.env['LOG_DUMP'] === 'false' ? false : true;	// when true or not specified, logs are also written to a log file in LOGS_DIR
 const DEBUG_MODE = process.env['DEBUG_MODE'] === 'true' ? true : false; // when true, temp folder is preserved
-const OUTPUT_DIR = 'output-' + new Date().getTime();
+const OUTPUT_DIR = 'output-' + TIME_SUFFIX;
 const DRY_RUN = false; // when true, terraform apply does not run
 
 
@@ -100,6 +101,16 @@ async function main(options) {
 		bearer = await getBearerToken(apiKey);
 		const accountId = await getAccountId(bearer, apiKey);
 
+		// check for continuous delivery instance in target region
+		if (!await getCdInstanceByRegion(bearer, accountId, targetRegion)) throw Error(`Could not find a Continuous Delivery instance in the target region '${targetRegion}', please create one before proceeding.`);
+
+		// check for existing .tf files in output directory
+		if (fs.existsSync(outputDir)) {
+			let files = fs.readdirSync(outputDir, { recursive: true });
+			files = files.filter((f) => f.endsWith('.tf'));
+			if (files.length > 0) throw Error(`Output directory already has ${files.length} '.tf' files, please specify a different output directory`);
+		}
+
 		if (options.gritMappingFile) {
 			gritMapping = JSON.parse(fs.readFileSync(resolve(options.gritMappingFile)));
 			const gritPromises = [];
@@ -238,7 +249,8 @@ async function main(options) {
 			LOG_STAGES.import
 		);
 
-		if (nonSecretRefs.length > 0) logger.warn(`\nWarning! The following generated terraform resource contains a hashed secret, applying without changes may result in error(s):\n${nonSecretRefs.map((entry) => `- ${entry}\n`).join('')}`, '', true);
+		if (nonSecretRefs.length > 0) logger.warn(`\nWarning! The following generated terraform resource contains hashed secret(s) that cannot be migrated, applying without changes may result in error(s):`);
+		logger.table(nonSecretRefs);
 
 	} catch (err) {
 		if (err.message && err.stack) {

package/cmd/utils/import-terraform.js

@@ -8,7 +8,6 @@
  */
 
 import fs from 'node:fs';
-import { promisify } from 'node:util';
 
 import { parse as tfToJson } from '@cdktf/hcl2json'
 import { jsonToTf } from 'json-to-tf';
@@ -19,8 +18,6 @@ import { getRandChars, isSecretReference, normalizeName } from './utils.js';
 
 import { SECRET_KEYS_MAP, SUPPORTED_TOOLS_MAP } from '../../config.js';
 
-const writeFilePromise = promisify(fs.writeFile);
-
 export async function importTerraform(token, apiKey, region, toolchainId, toolchainName, policyIds, dir, isCompact, verbosity) {
     // STEP 1/2: set up terraform file with import blocks
     const importBlocks = []; // an array of objects representing import blocks, used in importBlocksToTf
@@ -73,8 +70,15 @@ export async function importTerraform(token, apiKey, region, toolchainId, toolch
                     if (isSecretReference(tool.parameters[key])) {
                         additionalProps[block.name].push({ param: tfKey, value: tool.parameters[key] });
                     } else {
-                        nonSecretRefs.push(block.name);
-                        if (required) additionalProps[block.name].push({ param: tfKey, value: `<${tfKey}>` });
+                        const newFileName = SUPPORTED_TOOLS_MAP[tool.tool_type_id].split('ibm_')[1];
+                        if (required) {
+                            nonSecretRefs.push({
+                            resource_name: block.name, 
+                            property_name: tfKey,
+                            file_name: isCompact ? 'resources.tf' : `${newFileName}.tf`
+                            });
+                            additionalProps[block.name].push({ param: tfKey, value: `<${tfKey}>` });
+                        }
                     }
                 });
             }
@@ -150,7 +154,7 @@ export async function importTerraform(token, apiKey, region, toolchainId, toolch
         }
     }
 
-    await importBlocksToTf(importBlocks, dir);
+    importBlocksToTf(importBlocks, dir);
 
     if (!fs.existsSync(`${dir}/generated`)) fs.mkdirSync(`${dir}/generated`);
 
@@ -318,7 +322,7 @@ function importBlock(id, name, resourceType) {
 }
 
 // importBlocks array to tf file
-async function importBlocksToTf(blocks, dir) {
+function importBlocksToTf(blocks, dir) {
     let fileContent = '';
 
     blocks.forEach((block) => {
@@ -329,5 +333,5 @@ async function importBlocksToTf(blocks, dir) {
         fileContent += template;
     });
 
-    return await writeFilePromise(`${dir}/import.tf`, fileContent);
+    return fs.writeFileSync(`${dir}/import.tf`, fileContent);
 }

package/cmd/utils/requests.js

@@ -141,6 +141,31 @@ async function getToolchainsByName(bearer, accountId, toolchainName) {
     }
 }
 
+async function getCdInstanceByRegion(bearer, accountId, region) {
+    const apiBaseUrl = 'https://api.global-search-tagging.cloud.ibm.com/v3';
+    const options = {
+        url: apiBaseUrl + '/resources/search',
+        method: 'POST',
+        headers: {
+            'Authorization': `Bearer ${bearer}`,
+            'Content-Type': 'application/json',
+        },
+        data: {
+            'query': `service_name:continuous-delivery AND region:"${region}" AND doc.state:ACTIVE`,
+            'fields': ['doc.resource_group_id', 'doc.region_id']
+        },
+        params: { account_id: accountId },
+        validateStatus: () => true
+    };
+    const response = await axios(options);
+    switch (response.status) {
+        case 200:
+            return response.data.items.length > 0;
+        default:
+            throw Error('Get CD instance failed');
+    }
+}
+
 async function getToolchainTools(bearer, toolchainId, region) {
     const apiBaseUrl = `https://api.${region}.devops.cloud.ibm.com/toolchain/v2`;
     const options = {
@@ -388,6 +413,7 @@ async function deleteToolchain(bearer, toolchainId, region) {
 export {
     getBearerToken,
     getAccountId,
+    getCdInstanceByRegion,
     getToolchain,
     getToolchainsByName,
     getToolchainTools,

package/cmd/utils/terraform.js

@@ -9,7 +9,6 @@
 
 import child_process from 'node:child_process';
 import fs from 'node:fs';
-import { randomInt } from 'node:crypto';
 import { promisify } from 'node:util';
 
 import { parse as tfToJson } from '@cdktf/hcl2json'
@@ -302,7 +301,7 @@ async function setupTerraformFiles({ token, srcRegion, targetRegion, targetTag,
 }
 
 async function runTerraformPlanGenerate(dir, fileName) {
-    return await execPromise(`terraform plan -generate-config-out=${fileName}`, { cwd: dir });
+    return await execPromise(`terraform plan -generate-config-out="${fileName}"`, { cwd: dir });
 }
 
 async function runTerraformInit(dir) {

package/cmd/utils/validate.js

@@ -11,7 +11,7 @@ import { execSync } from 'child_process';
 import { logger, LOG_STAGES } from './logger.js'
 import { RESERVED_GRIT_PROJECT_NAMES, RESERVED_GRIT_GROUP_NAMES, RESERVED_GRIT_SUBGROUP_NAME, TERRAFORM_REQUIRED_VERSION, SECRET_KEYS_MAP } from '../../config.js';
 import { getToolchainsByName, getToolchainTools, getPipelineData, getAppConfigHealthcheck, getSecretsHealthcheck, getGitOAuth, getGritUserProject, getGritGroup, getGritGroupProject } from './requests.js';
-import { promptUserConfirmation, promptUserInput } from './utils.js';
+import { promptUserConfirmation, promptUserInput, isSecretReference } from './utils.js';
 
 
 function validatePrereqsVersions() {
@@ -146,7 +146,6 @@ async function validateTools(token, tcId, region, skipPrompt) {
     const toolsWithHashedParams = [];
     const patTools = [];
     const classicPipelines = [];
-    const secretPattern = /^hash:SHA3-512:[a-zA-Z0-9]{128}$/;
 
     for (const tool of allTools.tools) {
         const toolName = (tool.name || tool.parameters?.name || tool.parameters?.label || '').replace(/\s+/g, '+');
@@ -203,7 +202,7 @@ async function validateTools(token, tcId, region, skipPrompt) {
                 url: toolUrl
             });
         }
-        else if (['githubconsolidated', 'github_integrated', 'gitlab'].includes(tool.tool_type_id) && (tool.parameters?.auth_type === '' || tool.parameters?.auth_type === 'oauth')) {   // Skip secret check iff it's GitHub/GitLab integration with OAuth
+        else if (['githubconsolidated', 'github_integrated', 'gitlab', 'hostedgit'].includes(tool.tool_type_id) && (tool.parameters?.auth_type === '' || tool.parameters?.auth_type === 'oauth')) { // Skip secret check iff it's GitHub/GitLab/GRIT integration with OAuth
             continue;
         }
         else {
@@ -212,20 +211,23 @@ async function validateTools(token, tcId, region, skipPrompt) {
                 const pipelineData = await getPipelineData(token, tool.id, region);
 
                 pipelineData.properties.forEach((prop) => {
-                    if (prop.type === 'secure' && secretPattern.test(prop.value)) secrets.push(['properties', prop.name].join('.').replace(/\s+/g, '+'));
+                    if (prop.type === 'secure' && !isSecretReference(prop.value) && prop.value.length > 0)
+                        secrets.push(['properties', prop.name].join('.').replace(/\s+/g, '+'));
                 });
 
                 pipelineData.triggers.forEach((trigger) => {
-                    if ((trigger?.secret?.type === 'token_matches' || trigger?.secret?.type === 'digest_matches') && secretPattern.test(trigger.secret.value)) secrets.push([trigger.name, trigger.secret.key_name].join('.').replace(/\s+/g, '+'));
+                    if ((trigger?.secret?.type === 'token_matches' || trigger?.secret?.type === 'digest_matches') && !isSecretReference(trigger.secret.value) && trigger.secret.value.length > 0)
+                        secrets.push([trigger.name, trigger.secret.key_name].join('.').replace(/\s+/g, '+'));
                     trigger.properties.forEach((prop) => {
-                        if (prop.type === 'secure' && secretPattern.test(prop.value)) secrets.push([trigger.name, 'properties', prop.name].join('.').replace(/\s+/g, '+'));
+                        if (prop.type === 'secure' && !isSecretReference(prop.value) && prop.value.length > 0)
+                            secrets.push([trigger.name, 'properties', prop.name].join('.').replace(/\s+/g, '+'));
                     });
                 });
             }
             else {
                 const secretsToCheck = (SECRET_KEYS_MAP[tool.tool_type_id] || []).map((entry) => entry.key);    // Check for secrets in the rest of the tools
                 Object.entries(tool.parameters).forEach(([key, value]) => {
-                    if (secretPattern.test(value) && secretsToCheck.includes(key)) secrets.push(key);
+                    if (!isSecretReference(value) && value.length > 0 && secretsToCheck.includes(key)) secrets.push(key);
                 });
             }
             if (secrets.length > 0) {
@@ -260,7 +262,7 @@ async function validateTools(token, tcId, region, skipPrompt) {
     }
 
     if (toolsWithHashedParams.length > 0) {
-        logger.warn('Warning! The following tools contain secrets that cannot be migrated, please use the \'check-secret\' command to export the secrets: \n', LOG_STAGES.setup, true);
+        logger.warn('Warning! The following tools contain secrets that cannot be migrated, please use the \'check-secrets\' command to export the secrets: \n', LOG_STAGES.setup, true);
         logger.table(toolsWithHashedParams);
     }
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.2.1",
+  "version": "1.2.3",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",