@ibm-cloud/cd-tools 1.11.0 → 1.11.2

package/README.md

@@ -74,15 +74,20 @@ The `copy-project-group` command copies a [group](https://docs.gitlab.com/user/g
 $ npx @ibm-cloud/cd-tools copy-project-group -h
 Usage: @ibm-cloud/cd-tools copy-project-group [options]
 
-Bulk migrate GitLab group projects
+Copies all Git Repos and Issue Tracking projects in a group to another region.
+
+Examples:
+  npx @ibm-cloud/cd-tools copy-project-group -g "1796019" -s ca-tor -d us-south --st ${PAT_CA_TOR} --dt ${PAT_US_SOUTH}
+      Copy all the Git Repos and Issue Tracking projects in the group "mygroup" from the Toronto region to the Dallas, with the same group name.
 
 Options:
-  -s, --source-region <region>  Source GitLab instance region
-  -d, --dest-region <region>    Destination GitLab instance region
-  --st, --source-token <token>  Source GitLab access token
-  --dt, --dest-token <token>    Destination GitLab access token
-  -g, --group-id <id>           Source group ID to migrate
-  -n, --new-name <n>            New group path (optional)
+  -s, --source-region <region>  The source region from which to copy the project group (choices: "au-syd", "br-sao", "ca-mon", "ca-tor", "eu-de", "eu-es", "eu-gb", "jp-osa", "jp-tok", "us-east", "us-south")
+  -d, --dest-region <region>    The destination region to copy the projects to (choices: "au-syd", "br-sao", "ca-mon", "ca-tor", "eu-de", "eu-es", "eu-gb", "jp-osa", "jp-tok", "us-east", "us-south")
+  --st, --source-token <token>  A Git Repos and Issue Tracking personal access token from the source region. The api scope is required on the token.
+  --dt, --dest-token <token>    A Git Repos and Issue Tracking personal access token from the target region. The api scope is required on the token.
+  -g, --group-id <id>           The id of the group to copy from the source region (e.g. "1796019"), or the group name (e.g. "mygroup") for top-level groups. For sub-groups, a path
+                                is also allowed, e.g. "mygroup/subgroup"
+  -n, --new-group-slug <slug>   (Optional) Destination group URL slug (single path segment, e.g. "mygroup-copy"). Must be unique. Group display name remains the same as source.
   -h, --help                    display help for command
 ```
 
@@ -153,7 +158,7 @@ Advanced options:
   -d, --terraform-dir <path>             (Optional) The target local directory to store the generated Terraform (.tf) files
   -D, --dry-run                          (Optional) Skip running terraform apply; only generate the Terraform (.tf) files
   -f, --force                            (Optional) Force the copy toolchain command to run without user confirmation
-  -S, --skip-s2s                         (Optional) Skip importing toolchain-generated service-to-service authorizations
+  -S, --skip-s2s                         (Optional) Skip creating toolchain-generated service-to-service authorizations
   -T, --skip-disable-triggers            (Optional) Skip disabling Tekton pipeline Git or timed triggers. Note: This may result in duplicate pipeline runs
   -C, --compact                          (Optional) Generate all resources in a single resources.tf file
   -v, --verbose                          (Optional) Increase log output

package/cmd/direct-transfer.js

@@ -11,20 +11,24 @@ import { Command } from 'commander';
 import axios from 'axios';
 import readline from 'readline/promises';
 import { writeFile } from 'fs/promises';
-import { SOURCE_REGIONS } from '../config.js';
+import { COPY_PROJECT_GROUP_DESC, SOURCE_REGIONS } from '../config.js';
 import { getWithRetry } from './utils/requests.js';
 
 const HTTP_TIMEOUT_MS = 5 * 60 * 1000; // 5 minutes default
 
 class GitLabClient {
   constructor(baseURL, token) {
+    const root = baseURL.endsWith("/") ? baseURL : `${baseURL}/`;
+    
     this.client = axios.create({
-      baseURL: baseURL.endsWith('/') ? `${baseURL}api/v4` : `${baseURL}/api/v4`,
+      baseURL: `${root}api/v4`,
       timeout: HTTP_TIMEOUT_MS,
-      headers: {
-        'Authorization': `Bearer ${token}`,
-        'Content-Type': 'application/json'
-      }
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+    });
+    this.graph = axios.create({
+      baseURL: `${root}api`,
+      timeout: HTTP_TIMEOUT_MS,
+      headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
     });
   }
 
@@ -110,6 +114,50 @@ class GitLabClient {
     return projects;
   }
 
+  async listGroupProjectsGraphQL(groupFullPath, { includeSubgroups = true, pageSize = 200, maxProjects = 5000 } = {}) {
+    const out = [];
+    let after = null;
+
+    const query = `
+      query($fullPath: ID!, $after: String, $includeSubgroups: Boolean!, $pageSize: Int!) {
+        group(fullPath: $fullPath) {
+          projects(includeSubgroups: $includeSubgroups, first: $pageSize, after: $after) {
+            nodes {
+              fullPath
+              nameWithNamespace
+              httpUrlToRepo
+            }
+            pageInfo {
+              hasNextPage
+              endCursor
+            }
+          }
+        }
+      }
+    `;
+
+    while (out.length < maxProjects) {
+      const resp = await this.graph.post("/graphql", {
+        query,
+        variables: { fullPath: groupFullPath, after, includeSubgroups, pageSize },
+      });
+
+      if (resp.data?.errors?.length) {
+        throw new Error(`GraphQL errors: ${JSON.stringify(resp.data.errors)}`);
+      }
+
+      const projects = resp.data?.data?.group?.projects?.nodes || [];
+      const pageInfo = resp.data?.data?.group?.projects?.pageInfo;
+
+      out.push(...projects);
+
+      if (!pageInfo?.hasNextPage) break;
+      after = pageInfo.endCursor;
+    }
+
+    return out;
+  }
+
   async getGroup(groupId) {
     const response = await this.client.get(`/groups/${groupId}`);
     return response.data;
@@ -250,23 +298,19 @@ function validateAndConvertRegion(region) {
 }
 
 //  Build a mapping of: old http_url_to_repo -> new http_url_to_repo
-async function generateUrlMappingFile({ _, destUrl, sourceGroup, destinationGroupPath, sourceProjects }) {
+async function generateUrlMappingFile({ destUrl, sourceGroup, destinationGroupPath, sourceProjects }) {
   const destBase = destUrl.endsWith('/') ? destUrl.slice(0, -1) : destUrl;
   const urlMapping = {};
 
-  const groupPrefix = `${sourceGroup.full_path}/`;
-
   for (const project of sourceProjects) {
-    const oldRepoUrl = project.http_url_to_repo; // ends with .git
+    const oldRepoUrl = project.http_url_to_repo || project.httpUrlToRepo;
 
-    // path_with_namespace is like "group/subgroup/project-1"
-    let relativePath;
-    if (project.path_with_namespace.startsWith(groupPrefix)) {
-      relativePath = project.path_with_namespace.slice(groupPrefix.length);
-    } else {
-      // Fallback if for some reason full_path is not a prefix
-      relativePath = project.path_with_namespace;
-    }
+    const fullPath = project.path_with_namespace || project.fullPath || "";
+    const groupPrefix = `${sourceGroup.full_path}/`;
+
+    const relativePath = fullPath.startsWith(groupPrefix)
+      ? fullPath.slice(groupPrefix.length)
+      : fullPath;
 
     const newRepoUrl = `${destBase}/${destinationGroupPath}/${relativePath}.git`;
     urlMapping[oldRepoUrl] = newRepoUrl;
@@ -436,7 +480,7 @@ async function handleBulkImportConflict({ destination, destUrl, sourceGroupFullP
         }
 
         console.log(`\nConflict detected: ${importResErr}`);
-        console.log(`Please specify a new group name using -n, --new-name <n> when trying again`);
+        console.log(`Please specify a new group name using -n, --new-group-slug <n> when trying again`);
         console.log(`\nGroup already migrated.`);
         if (groupUrl) console.log(`Migrated group: ${groupUrl}`);
         if (historyUrl) console.log(`Group import history: ${historyUrl}`);
@@ -463,23 +507,32 @@ async function directTransfer(options) {
     console.log(`Fetching source group from ID: ${options.groupId}...`);
     const sourceGroup = await source.getGroup(options.groupId);
 
-    // let destinationGroupName = options.newName || sourceGroup.name;
-    let destinationGroupPath = options.newName || sourceGroup.path;
+    let destinationGroupPath = options.newGroupSlug || sourceGroup.path;
 
-    const sourceProjects = await source.getGroupProjects(sourceGroup.id);
+    let sourceProjects;
+    try {
+      sourceProjects = await source.listGroupProjectsGraphQL(sourceGroup.full_path, {
+        includeSubgroups: true,
+        pageSize: 100,
+        maxProjects: 10000,
+      });
+    } catch (e) {
+      console.warn(`[WARN] GraphQL listing failed (${e.message}). Falling back to REST safe listing...`);
+      sourceProjects = await source.getGroupProjects(sourceGroup.id);
+    }
+    
     console.log(`Found ${sourceProjects.length} projects in source group`);
     if (sourceProjects.length > 0) {
       console.log('Projects to be migrated:');
-      sourceProjects.forEach(p => console.log(`${p.name_with_namespace}`));
+      sourceProjects.forEach(p => console.log(p.name_with_namespace || p.nameWithNamespace || p.fullPath));
     }
 
-    if (options.newName) {
-      await promptUser(options.newName);
+    if (options.newGroupSlug) {
+      await promptUser(options.newGroupSlug);
     }
 
     // Generate URL mapping JSON before starting the migration
     await generateUrlMappingFile({
-      sourceUrl,
       destUrl,
       sourceGroup,
       destinationGroupPath,
@@ -624,13 +677,14 @@ async function directTransfer(options) {
 }
 
 const command = new Command('copy-project-group')
-  .description('Bulk migrate GitLab group projects')
-  .requiredOption('-s, --source-region <region>', 'Source GitLab instance region')
-  .requiredOption('-d, --dest-region <region>', 'Destination GitLab instance region')
-  .requiredOption('--st, --source-token <token>', 'Source GitLab access token')
-  .requiredOption('--dt, --dest-token <token>', 'Destination GitLab access token')
-  .requiredOption('-g, --group-id <id>', 'Source group ID to migrate')
-  .option('-n, --new-name <n>', 'New group path (optional)')
+  .summary('Copies all Git Repos and Issue Tracking projects in a group to another region.')
+  .description(COPY_PROJECT_GROUP_DESC)
+  .requiredOption('-s, --source-region <region>', 'The source region from which to copy the project group (choices: "au-syd", "br-sao", "ca-mon", "ca-tor", "eu-de", "eu-es", "eu-gb", "jp-osa", "jp-tok", "us-east", "us-south")')
+  .requiredOption('-d, --dest-region <region>', 'The destination region to copy the projects to (choices: "au-syd", "br-sao", "ca-mon", "ca-tor", "eu-de", "eu-es", "eu-gb", "jp-osa", "jp-tok", "us-east", "us-south")')
+  .requiredOption('--st, --source-token <token>', 'A Git Repos and Issue Tracking personal access token from the source region. The api scope is required on the token.')
+  .requiredOption('--dt, --dest-token <token>', 'A Git Repos and Issue Tracking personal access token from the target region. The api scope is required on the token.')
+  .requiredOption('-g, --group-id <id>', 'The id of the group to copy from the source region (e.g. "1796019"), or the group name (e.g. "mygroup") for top-level groups. For sub-groups, a path is also allowed, e.g. "mygroup/subgroup"')
+  .option('-n, --new-group-slug <slug>', '(Optional) Destination group URL slug (single path segment, e.g. "mygroup-copy"). Must be unique. Group display name remains the same as source.')
   .showHelpAfterError()
   .hook('preAction', cmd => cmd.showHelpAfterError(false)) // only show help during validation
   .action(async (options) => {

package/config.js

@@ -17,7 +17,13 @@ Examples:
       Copy a toolchain to the Frankfurt region with the specified name and target resource group, using the given API key
 
 Environment Variables:
-  IBMCLOUD_API_KEY                       API key used to authenticate. Must have IAM permission to read and create toolchains and service-to-service authorizations in source and target region / resource group`
+  IBMCLOUD_API_KEY                       API key used to authenticate. Must have IAM permission to read and create toolchains and service-to-service authorizations in source and target region / resource group`;
+
+const COPY_PROJECT_GROUP_DESC = `Copies all Git Repos and Issue Tracking projects in a group to another region.
+
+Examples:
+  npx @ibm-cloud/cd-tools copy-project-group -g "1796019" -s ca-tor -d us-south --st \${PAT_CA_TOR} --dt \${PAT_US_SOUTH}
+      Copy all the Git Repos and Issue Tracking projects in the group "mygroup" from the Toronto region to the Dallas, with the same group name.`;
 
 const DOCS_URL = 'https://github.com/IBM/continuous-delivery-tools';
 
@@ -222,6 +228,7 @@ const VAULT_REGEX = [
 
 export {
 	COPY_TOOLCHAIN_DESC,
+	COPY_PROJECT_GROUP_DESC,
 	DOCS_URL,
 	SOURCE_REGIONS,
 	TARGET_REGIONS,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ibm-cloud/cd-tools",
-  "version": "1.11.0",
+  "version": "1.11.2",
   "description": "Tools and utilities for the IBM Cloud Continuous Delivery service and resources",
   "repository": {
     "type": "git",