@docupace/pipeline 1.0.10 → 1.0.12

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docupace/pipeline",
-  "version": "1.0.10",
+  "version": "1.0.12",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -24,7 +24,7 @@
     "lodash": "^4.17.21",
     "openai": "^4.76.3",
     "zod": "^3.24.1",
-    "zx": "^8.2.4"
+    "zx": "^8.3.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.17.0",

package/src/auth0/configure.js

@@ -164,28 +164,28 @@ async function templateAuth0(siteUrl, coreApiContext, env, filePaths, deployUI,
 
     // Configure URLs based on environment
     if (env === 'prd') {
-      uiAllowedCallbacksUrls = [`https://${siteUrl}/api/auth/callback`];
-      uiAllowedLogoutUrls = [`https://${siteUrl}`];
+      uiAllowedCallbacksUrls = [`${siteUrl}/api/auth/callback`];
+      uiAllowedLogoutUrls = [`${siteUrl}`];
 
-      graphqlAllowedCallbacksUrls = [`https://${siteUrl}/api/graphql/callback`];
-      graphqlAllowedLogoutUrls = [`https://${siteUrl}/api/graphql/`];
+      graphqlAllowedCallbacksUrls = [`${siteUrl}/api/graphql/callback`];
+      graphqlAllowedLogoutUrls = [`${siteUrl}/api/graphql/`];
     } else {
       uiAllowedCallbacksUrls = [
-        `https://${siteUrl}/api/auth/callback`,
+        `${siteUrl}/api/auth/callback`,
         `http://localhost:3000/api/auth/callback`,
       ];
       uiAllowedLogoutUrls = [
-        `https://${siteUrl}`,
+        `${siteUrl}`,
         `http://localhost:3000`,
       ];
 
       graphqlAllowedCallbacksUrls = [
-        `https://${siteUrl}/api/graphql/callback`,
+        `${siteUrl}/api/graphql/callback`,
         `http://localhost:4000/api/graphql/callback`,
       ];
       graphqlAllowedLogoutUrls = [
         `http://localhost:4000/api/graphql/`,
-        `https://${siteUrl}/api/graphql/`,
+        `${siteUrl}/api/graphql/`,
       ];
     }
 

package/src/next/apache.js

@@ -0,0 +1,36 @@
+import { $ } from 'zx';
+
+export async function addApacheSite() {
+    if (
+      !process.env.ENVIRONMENT ||
+      (process.env.ENVIRONMENT !== 'dev' && process.env.ENVIRONMENT !== 'qa')
+    ) {
+      log("Error: ENVIRONMENT must be either 'dev' or 'qa'.");
+      process.exit(1);
+    }
+    if (!process.env.SITE) {
+      log('Error: SITE cannot be empty.');
+      process.exit(1);
+    }
+  
+    // Save private key from base64
+    const haproxyPrivateKey = process.env.HAPROXY_PRIVATE_KEY;
+    if (!haproxyPrivateKey) {
+      log('Error: HAPROXY_PRIVATE_KEY not set.');
+      process.exit(1);
+    }
+  
+    await fs.writeFile('private.key', Buffer.from(haproxyPrivateKey, 'base64'));
+    await fs.chmod('private.key', 0o600);
+  
+    const server = `${process.env.ENVIRONMENT}-apalb01a.corp.docupace.com`;
+    console.log("Running command");
+    const sshCmd = (script) =>
+      $`ssh -i private.key -o StrictHostKeyChecking=no root@${server} ${script}`;
+  
+    // The script is run twice in the Bash version
+    await sshCmd(`/root/repo/iis-preprod/scripts/addsite/add_core_site.sh ${process.env.ENVIRONMENT} ${process.env.SITE}`);
+    //await sshCmd(`/root/repo/iis-preprod/scripts/addsite/add_core_site.sh ${process.env.ENVIRONMENT} ${process.env.SITE}`);
+  }
+  
+  

package/src/next/cli.js

@@ -7,6 +7,10 @@ import { publish } from './publish.js';
 import { deployFromSite } from './deploy.js';
 import { deployTemplate } from './deploy.js';
 import { copy } from './copy.js';
+import { vaultAddSiteData } from './vault.js';
+import { addApacheSite } from './apache.js';
+import { haproxySync }  from './haproxy.js';
+import { connectClone } from './connect.js';
 import dotenv from 'dotenv';
 
 dotenv.config({ path: ['.env.pipeline.local', 'config/.env.pipeline.local'] });
@@ -17,6 +21,7 @@ const program = new Command();
 import { $ } from 'zx';
 import {deployConnect, deployCore} from './jenkins.js';
 import { runPipeline } from './pipline.js';
+
 const GIT_SHA = (await $`git log --pretty=format:'%h' -n 1`).stdout.trim();
 
 program
@@ -33,6 +38,39 @@ program
     await sonar(GIT_SHA);
   });
 
+program
+.command('addApacheSite')
+.description('Add site to apache')
+.action(async () => {
+  await addApacheSite();
+});
+
+program
+.command('connectClone')
+.description('Clone connect site')
+.action(async () => {
+  await connectClone();
+});
+
+
+program
+.command('haproxySync')
+.description('Sync haproxy config')
+.action(async () => {
+  await haproxySync();
+});
+
+
+
+program
+  .command('vaultAddSiteData')
+  .description('Add m2m auth0 data to vault')
+  .requiredOption('-s, --site-name <siteName>', 'Site name')
+  .action(async (options) => {
+    const { siteName,  } = options;
+    await vaultAddSiteData(siteName);
+  });
+
 program
   .command('publish')
   .description('Publish a Docker image')
@@ -92,7 +130,8 @@ program
     });
   });
 
-program
+
+  program
   .command('copy')
   .description('Copy a Docker image between registries')
   .requiredOption('-a, --application <application>', 'Application name')

package/src/next/connect.js

@@ -0,0 +1,190 @@
+import 'zx/globals'; // npm i zx
+import fs from 'fs-extra';
+import path from 'path';
+import axios from 'axios';
+import { $ } from 'zx';
+
+
+
+import {
+  cloneEnvRepository,
+  cloneRepository
+} from './deploy.js';
+
+
+import { getVaultSecrets } from '../auth0/vault.js';
+
+import {
+  getVaultSecretsConfigs,
+  environmentConfig
+} from './config.js';
+
+process.env.COMMON_VAULT_PATH = process.env.COMMON_VAULT_PATH || 'secret/data/common/config';
+process.env.VAULT_SKIP_VERIFY = process.env.VAULT_SKIP_VERIFY || 'true';
+process.env.CONNECT_DIR = process.env.CONNECT_DIR || 'connect';
+const vaultToken = process.env.VAULT_TOKEN;
+const vaultAddr = process.env.VAULT_ADDR;
+
+function log(msg) {
+    console.log(`[*] ${msg}`);
+  }
+  
+  /* --------------------------------
+   * 1. GIT CHECKOUT
+   * -------------------------------- */
+  async function gitCheckout(repository, destDir, branch = 'master') {
+    if (!(await fs.pathExists(destDir))) {
+      log(`Directory ${destDir} does not exist. Cloning repository...`);
+    } else {
+      log(`Directory ${destDir} exists. Removing it...`);
+      await fs.remove(destDir);
+      log(`Cloning ${destDir} repository...`);
+    }
+    
+    const username = process.env.BITBUCKET_USERNAME;
+    const password = process.env.BITBUCKET_APP_PASSWORD;
+  
+    // Construct clone command
+    const cloneUrl = `https://${username}:${password}@bitbucket.org/dp-cloud/${repository}.git`;
+  
+    log(`Perform cloning`);
+    await $`git clone --branch=${branch} --depth 1 ${cloneUrl} ${destDir}`;
+  }
+  
+  
+
+
+  
+
+export async function connectClone() {
+
+
+    if (!vaultToken || !vaultAddr) {
+      console.error("Error: VAULT_TOKEN and VAULT_ADDR must be set.");
+      throw new Error("VAULT_TOKEN and VAULT_ADDR must be set.");
+    }
+
+    const site = process.env.BUILD_SITE;
+    if (!site) {
+      log('Error: BUILD_SITE is not set');
+      process.exit(1);
+    }
+    const siteCommonVaultPath = `secret/data/${site}/common/config`;
+  
+  
+    // 2) getVaultSecrets
+    const secretsJson = await getVaultSecrets(siteCommonVaultPath);
+    if (!secretsJson.AUTH0_CLIENT_ID) {
+      log('Failed to retrieve secret data.');
+      process.exit(1);
+    }
+    const siteCommonSecrets = secretsJson;
+  
+    // 3) Export secrets as environment variables
+    process.env.AUTH0_CLIENT_ID = siteCommonSecrets.AUTH0_CLIENT_ID;
+    process.env.AUTH0_CLIENT_SECRET = siteCommonSecrets.AUTH0_CLIENT_SECRET;
+  
+   
+    process.env.AUTH0_REGION_DOMAIN="us.auth0.com";
+
+    log(`BUILD_ENVIRONMENT: ${process.env.BUILD_ENVIRONMENT}`);
+    log(`BUILD_SITE: ${process.env.BUILD_SITE}`);
+    log(`AUTH0_SUBDOMAIN: ${process.env.AUTH0_SUBDOMAIN}`);
+    log(`AUTH0_REGION_DOMAIN: ${process.env.AUTH0_REGION_DOMAIN}`);
+    log(`AUTH0_CLIENT_ID: ${process.env.AUTH0_CLIENT_ID}`);
+  
+    process.env.AUTH0_DOMAIN = `docupace-${process.env.BUILD_ENVIRONMENT}-${process.env.BUILD_SITE}.${process.env.AUTH0_REGION_DOMAIN}`;
+    
+
+    const projectRoot = process.cwd();
+    // 4) Clone 'connect-deploy' into CONNECT_DIR
+
+    
+    await gitCheckout('connect-deploy', process.env.CONNECT_DIR);
+    await gitCheckout(`${process.env.BUILD_ENVIRONMENT}-site-settings`, `${process.env.CONNECT_DIR}/${process.env.BUILD_ENVIRONMENT}-site-settings`);
+
+    // 5) Create site clone (emulating the dock.sh call)
+    const cloneFrom = path.join(projectRoot,process.env.CONNECT_DIR,`envs`,`${process.env.FROM_BUILD_ENVIRONMENT}/${process.env.FROM_BUILD_SITE}`);
+    const cloneTo = path.join(projectRoot,process.env.CONNECT_DIR,`envs`,`${process.env.BUILD_ENVIRONMENT}/${process.env.BUILD_SITE}`);
+
+    const connectHome = path.join(projectRoot,process.env.CONNECT_DIR);
+    if (!(await fs.pathExists(cloneFrom))) {
+      log('Clone template directory not found:');
+      process.exit(1);
+    }
+  
+  
+    if (!(await fs.pathExists(cloneTo))) {
+      log('New site does not exist. Performing cloning...');
+
+      process.env.IMAGE_VERSION=await $`cd ${cloneFrom} && grep -E '^IMAGE_VERSION=' dock.sh | cut -f2 -d=`
+      log(`Image version: ${process.env.IMAGE_VERSION}` );
+      //await $`cd ${cloneFrom} && bash dock.sh clone ${process.env.BUILD_ENVIRONMENT} ${process.env.BUILD_SITE} ${process.env.IMAGE_VERSION}`;
+      await (async () => {
+        try {
+            // Execute the shell command
+            const result = await $`cd ${cloneFrom} && bash dock.sh clone ${process.env.BUILD_ENVIRONMENT} ${process.env.BUILD_SITE} ${process.env.IMAGE_VERSION}`;
+            console.log('Command succeeded:');
+            console.log('stdout:', result.stdout); // Standard output
+            console.log('stderr:', result.stderr); // Standard error
+        } catch (error) {
+            console.error('Command failed:');
+            console.error('Exit Code:', error._code); // Exit code of the command
+            console.error('Signal:', error._signal); // Signal that caused termination (if any)
+            console.error('stdout:', error._stdout); // Standard output
+            console.error('stderr:', error._stderr); // Standard error
+        }
+    })();
+    
+    
+    } else {
+      log('Site already exists. Continuing Auth0 setup...');
+    }
+  
+    
+    await (async () => {
+      try {
+          // Execute the shell command
+          const result = await $`cd ${cloneTo} && bash dock.sh getAuth0Metadata`;
+          console.log('Command succeeded:');
+          console.log('stdout:', result.stdout); // Standard output
+          console.log('stderr:', result.stderr); // Standard error
+      } catch (error) {
+          console.error('Command failed:');
+          console.error('Exit Code:', error._code); // Exit code of the command
+          console.error('Signal:', error._signal); // Signal that caused termination (if any)
+          console.error('stdout:', error._stdout); // Standard output
+          console.error('stderr:', error._stderr); // Standard error
+      }
+  })();
+
+
+    // 7) Check conf files, print logs
+    await $`grep "auth0" ${path.join(cloneTo, 'conf/application.conf')}`;
+    await $`ls -all ${path.join(cloneTo, 'conf')}`;
+    await $`ls -all ${path.join(cloneTo, 'conf/saml')}`;
+    await $`cat ${path.join(cloneTo, 'conf/saml/auth0.xml')}`;
+  
+    // 8) Git add, commit, push
+    await $`cd ${cloneTo} && git checkout master && git pull --ff-only`;
+    await $`cd ${cloneTo} && git remote -v`;
+    await $`cd ${cloneTo} && git branch`;
+    await $`cd ${cloneTo} && git status`;
+    await $`cd ${cloneTo} && git add . ../sites.sh`;
+    await $`cd ${cloneTo} && git status`;
+  
+    const numChanges = (
+      await $`cd ${cloneTo} && git diff-index HEAD`
+    ).stdout.split('\n').filter(Boolean).length;
+  
+    if (numChanges > 0) {
+      const jiraTicket = process.env.JIRA_TICKET || 'NO_JIRA';
+      const message = `${jiraTicket} usdoc-s-builui job download_auth0_metadata for ${process.env.BUILD_ENVIRONMENT}/${process.env.BUILD_SITE}`;
+      log('About to commit changes to git');
+      await $`cd ${cloneTo} && git commit -m ${message}`;
+      await $`cd ${cloneTo} && git push -u origin master`;
+    } else {
+      log('No changes have been made. The site likely already has the correct configuration.');
+    }
+  }
+  

package/src/next/haproxy.js

@@ -0,0 +1,50 @@
+import { $ } from 'zx';
+
+
+export async function haproxySync() {
+    const repoPath = '/root/repo/docupace-ansible';
+    const inventory = path.join(repoPath, 'inventories/preprod/inventory');
+  
+    // Create haproxy.sh
+    const scriptContent = `
+  #!/bin/bash
+  
+  [[ ! -f ${inventory} ]] && ( echo "Alert: no inventory file: ${inventory}" ; exit 1 )
+  cd ${repoPath}
+  pwd
+  git stash
+  git pull origin master
+  git checkout master
+  git submodule update --remote
+  git submodule foreach --recursive git checkout master
+  git submodule foreach --recursive git pull origin master
+  
+  ansible-playbook -i inventories/preprod/inventory haproxy.yml -e "cluster=preprod"
+  
+  echo "proceeding"
+  echo "committing changes"
+  git add preprod-haproxy.cfg
+  git commit -m "Sync haproxy"
+  git push origin master
+  `;
+  
+    await fs.writeFile('haproxy.sh', scriptContent);
+  
+    // Private key
+    const haproxyPrivateKey = process.env.HAPROXY_PRIVATE_KEY;
+    if (!haproxyPrivateKey) {
+      log('Error: HAPROXY_PRIVATE_KEY not set.');
+      process.exit(1);
+    }
+  
+    await fs.writeFile('private.key', Buffer.from(haproxyPrivateKey, 'base64'));
+    await fs.chmod('private.key', 0o600);
+  
+    await $`cat private.key`; // Just to mimic the script's behavior
+  
+    // Copy script to remote
+    await $`scp -i private.key -o StrictHostKeyChecking=no haproxy.sh root@10.10.28.198:/tmp/haproxy.sh`;
+  
+    // Execute script on remote
+    await $`ssh -i private.key -o StrictHostKeyChecking=no root@10.10.28.198 "bash /tmp/haproxy.sh"`;
+  }

package/src/next/vault.js

@@ -0,0 +1,39 @@
+import axios from 'axios';
+
+export async function vaultAddSiteData(SITE) {
+    const AUTH0_CLIENT_ID = process.env.AUTH0_CLIENT_ID;
+    const AUTH0_CLIENT_SECRET = process.env.AUTH0_CLIENT_SECRET;
+    const VAULT_ADDR = process.env.VAULT_ADDR;       // Vault host URL from environment variables
+    const VAULT_TOKEN = process.env.VAULT_TOKEN;     // Vault token from environment variables
+
+    if (!VAULT_ADDR || !VAULT_TOKEN) {
+        throw new Error('Missing VAULT_ADDR or VAULT_TOKEN environment variable.');
+    }
+
+    const vaultApiUrl = `${VAULT_ADDR}/v1/secret/data/${SITE}/common/config`;
+
+    try {
+        const response = await axios.post(
+            vaultApiUrl,
+            {
+                data: {
+                    AUTH0_CLIENT_ID: AUTH0_CLIENT_ID,
+                    AUTH0_CLIENT_SECRET: AUTH0_CLIENT_SECRET,
+                },
+            },
+            {
+                headers: {
+                    'X-Vault-Token': VAULT_TOKEN,
+                    'Content-Type': 'application/json',
+                },
+            }
+        );
+
+        console.log('Data added to Vault successfully:', response.data);
+    } catch (error) {
+        console.error('Error adding data to Vault:', error.response?.data || error.message);
+        throw error;
+    }
+}
+
+

package/terraform/apply.sh

@@ -0,0 +1,26 @@
+#!/usr/bin/env bash
+set -e
+
+PROJECT=$1
+
+if [[ -z "$PROJECT" ]]; then
+  echo "Error: Project name is required."
+  echo "Usage: $0 <project-name>"
+  exit 1
+fi
+
+export ROLE_NAME="atlassian-bitbucket-pipelines-runner-eks-role"
+export SESSION_NAME="EKSAdminSession"
+export EKS_CLUSTER_NAME="connect-next-${CURRENT_ENVIRONMENT}-eks-cluster"
+export ROLE_ARN="arn:aws:iam::$(aws sts get-caller-identity | jq -r '.Account'):role/$ROLE_NAME"
+export ASSUME_ROLE_OUTPUT=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name $SESSION_NAME)
+export AWS_ACCESS_KEY_ID=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.AccessKeyId')
+export AWS_SECRET_ACCESS_KEY=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SecretAccessKey')
+export AWS_SESSION_TOKEN=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SessionToken')
+export TF_IN_AUTOMATION=1
+
+aws sts get-caller-identity
+
+cd "$BITBUCKET_CLONE_DIR/${PROJECT}"
+
+terraform apply -input=false .tfplan

package/terraform/init.sh

@@ -0,0 +1,36 @@
+#!/bin/bash
+set -e
+
+PROJECT=$1
+CURRENT_ENVIRONMENT=$2
+
+if [[ -z "$PROJECT" ]]; then
+  echo "Error: Project name is required."
+  echo "Usage: $0 <project-name> $1 <environment>"
+  exit 1
+fi
+
+if [[ -z "$CURRENT_ENVIRONMENT" ]]; then
+  echo "Error: ENVIRONMENT is required."
+  echo "Usage: $0 <project-name> $1 <environment>"
+  exit 1
+fi
+
+export ROLE_NAME="atlassian-bitbucket-pipelines-runner-eks-role"
+export SESSION_NAME="EKSAdminSession"
+export EKS_CLUSTER_NAME="connect-next-${CURRENT_ENVIRONMENT}-eks-cluster"
+export ROLE_ARN="arn:aws:iam::$(aws sts get-caller-identity | jq -r '.Account'):role/$ROLE_NAME"
+export ASSUME_ROLE_OUTPUT=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name $SESSION_NAME)
+export AWS_ACCESS_KEY_ID=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.AccessKeyId')
+export AWS_SECRET_ACCESS_KEY=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SecretAccessKey')
+export AWS_SESSION_TOKEN=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SessionToken')
+
+aws sts get-caller-identity
+cd "$BITBUCKET_CLONE_DIR/${PROJECT}"
+mkdir -p ~/.ssh
+
+echo "$atlassian_workspace_runner_ssh_key" | base64 -d > ~/.ssh/id_ed25519
+chmod 600 ~/.ssh/id_ed25519
+ssh-keyscan bitbucket.org > ~/.ssh/known_hosts
+
+terraform init -backend-config=./env/${CURRENT_ENVIRONMENT}/us-west-2/backend.tfvars --reconfigure

package/terraform/plan.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+set -e
+
+PROJECT=$1
+CURRENT_ENVIRONMENT=$2
+
+if [[ -z "$PROJECT" ]]; then
+  echo "Error: Project name is required."
+  echo "Usage: $0 <project-name> $1 <environment>"
+  exit 1
+fi
+
+if [[ -z "$CURRENT_ENVIRONMENT" ]]; then
+  echo "Error: ENVIRONMENT is required."
+  echo "Usage: $0 <project-name> $1 <environment>"
+  exit 1
+fi
+
+export ROLE_NAME="atlassian-bitbucket-pipelines-runner-eks-role"
+export SESSION_NAME="EKSAdminSession"
+export EKS_CLUSTER_NAME="connect-next-${CURRENT_ENVIRONMENT}-eks-cluster"
+export ROLE_ARN="arn:aws:iam::$(aws sts get-caller-identity | jq -r '.Account'):role/$ROLE_NAME"
+export ASSUME_ROLE_OUTPUT=$(aws sts assume-role --role-arn $ROLE_ARN --role-session-name $SESSION_NAME)
+export AWS_ACCESS_KEY_ID=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.AccessKeyId')
+export AWS_SECRET_ACCESS_KEY=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SecretAccessKey')
+export AWS_SESSION_TOKEN=$(echo $ASSUME_ROLE_OUTPUT | jq -r '.Credentials.SessionToken')
+
+aws sts get-caller-identity
+
+cd "$BITBUCKET_CLONE_DIR/${PROJECT}"
+
+if [[ "$PROJECT" == "bb-build-agent-shell" ]]; then
+  if [[ "$CURRENT_ENVIRONMENT" == "dev" ]]; then
+    echo "$dev_build_agent_configuration" | base64 -d > ./env/${CURRENT_ENVIRONMENT}/us-west-2/build_agent_configuration.tfvars.json
+  elif [[ "$CURRENT_ENVIRONMENT" == "qa" ]]; then
+    echo "$qa_build_agent_configuration" | base64 -d > ./env/${CURRENT_ENVIRONMENT}/us-west-2/build_agent_configuration.tfvars.json
+  fi
+
+  terraform plan -var-file="./env/${CURRENT_ENVIRONMENT}/us-west-2/terraform.tfvars" \
+                 -var-file="./env/${CURRENT_ENVIRONMENT}/us-west-2/build_agent_configuration.tfvars.json" \
+                 -out .tfplan
+else
+  terraform plan -var-file="./env/${CURRENT_ENVIRONMENT}/us-west-2/terraform.tfvars" -out .tfplan
+fi

package/terraform/validate.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+set -e
+
+PROJECT=$1
+
+if [[ -z "$PROJECT" ]]; then
+  echo "Error: Project name is required."
+  echo "Usage: $0 <project-name>"
+  exit 1
+fi
+
+cd "$BITBUCKET_CLONE_DIR/${PROJECT}"
+
+terraform version
+terraform init -backend=false
+terraform validate
+terraform fmt -check -recursive --diff