@docupace/pipeline 1.0.7-alpha.9 → 1.0.7-alpha.ervins1.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@docupace/pipeline",
-  "version": "1.0.7-alpha.9",
+  "version": "1.0.7-alpha.ervins1.1",
   "description": "",
   "main": "index.js",
   "type": "module",
@@ -24,7 +24,7 @@
     "lodash": "^4.17.21",
     "openai": "^4.76.3",
     "zod": "^3.24.1",
-    "zx": "^8.2.4"
+    "zx": "^8.3.0"
   },
   "devDependencies": {
     "@eslint/js": "^9.17.0",

package/src/auth0/configure.js

@@ -32,6 +32,38 @@ async function ensureDirectoryExists(dirPath) {
   }
 }
 
+async function addApplicationToApi ({application, auth0Domain, auth0AuthToken, coreApiIdentifier}){
+  console.log("Adding application to API authorized applications...");
+
+  try {
+    const auth0Response = await axios.get(`https://${auth0Domain}/api/v2/clients`, {
+      headers: {authorization: `Bearer ${auth0AuthToken}`},
+    });
+
+    const appAuth0ClientIdData = auth0Response.data.find(client => client.name === `${application}`);
+    const appAuth0ClientId = appAuth0ClientIdData.client_id;
+
+    await axios.post(
+          `https://${auth0Domain}/api/v2/client-grants`,
+      {
+        client_id: appAuth0ClientId,
+        audience: `${coreApiIdentifier}`,
+        scope: [],
+      },
+      {
+        headers: {
+          Authorization: `Bearer ${auth0AuthToken}`,
+          "Content-Type": "application/json",
+        },
+      }
+    );
+    console.log(`Application (${appAuth0ClientId}) authorized for API: ${coreApiIdentifier}`);
+  } catch (err) {
+    console.error(`Failed to add application: ${application}: ${err.message}`);
+  }
+
+}
+
 async function initializeAuth0Configuration({
                                               initConfiguration,
                                               auth0Domain,
@@ -63,7 +95,6 @@ async function initializeAuth0Configuration({
       );
 
       const apiId = resourceServerResponse.data.id;
-      console.log(`API ID: ${apiId}`);
 
       // Update tenant settings with the default audience
       console.log("Updating tenant settings...");
@@ -79,7 +110,6 @@ async function initializeAuth0Configuration({
           },
         }
       );
-      console.log(`coreApiIdentifier => ${coreApiIdentifier}`)
       console.log("Tenant settings updated successfully.");
     } catch (error) {
       console.error("Error during Auth0 configuration:", error.response?.data || error.message);
@@ -372,7 +402,7 @@ externalConnectService:
   }
 }
 
-async function configureGraphQLServer(siteHost, coreURL, docupaceHost, site, env, auth0Domain, auth0ClientId, auth0ClientSecret, auth0Token, vaultSiteSecretsPath, imageTag, shortTemplateLatestCommitId, repoPath, commonSecrets) {
+async function configureGraphQLServer(siteHost, coreURL, docupaceHost, site, env, auth0Domain, auth0ClientId, auth0ClientSecret, auth0Token, vaultSiteSecretsPath, imageTag, standardConfigRemoteBranch, repoPath, commonSecrets) {
   console.log(`Configuring GraphQL Server for site: ${site} Host: ${siteHost} in env: ${env}`);
 
   try {
@@ -408,7 +438,7 @@ async function configureGraphQLServer(siteHost, coreURL, docupaceHost, site, env
     // Prepare ui.app.yaml
     const graphQLServerAppYaml = `
 repo: docupace-graphql-server
-revision: master
+revision: ${standardConfigRemoteBranch}
 `.trim();
 
     // Prepare ui.values.yaml
@@ -420,12 +450,10 @@ graphqlServer:
   baseUrl: https://${siteHost}/api/graphql
   coreUrl: https://${coreURL}
   integrationHubUrl: "https://${commonSecrets.INTEGRATION_HUB_URL}"
-  standardConfigRemoteBranch: ${shortTemplateLatestCommitId}
+  standardConfigRemoteBranch: ${standardConfigRemoteBranch}
   apiLevel: "2"
   env:
     AUTH0_CLIENT_SECRET: vault:secret/data/${site}/graphql-server/config#AUTH0_CLIENT_SECRET
-    CORE_PASSWORD: vault:secret/data/common/config#CORE_PASSWORD
-    REMOTE_CONFIG_PATH: vault:secret/data/common/config#REMOTE_CONFIG_PATH
     STANDARD_CONFIG_REMOTE_PATH: vault:secret/data/common/config#STANDARD_CONFIG_REMOTE_PATH
     AUTH0_CC_CLIENT_ID: vault:secret/data/${site}/graphql-server/config#AUTH0_CC_CLIENT_ID
     AUTH0_CC_CLIENT_SECRET: vault:secret/data/${site}/graphql-server/config#AUTH0_CC_CLIENT_SECRET
@@ -514,6 +542,7 @@ export async function configureFromSite(fromSite, toSite, env, deployUI, deployG
 
   const { deployConfig: deployGraphQLServerConfig} = await readDeployConfig('graphql-server', env, fromSite);
   const graphqlImageTag = deployGraphQLServerConfig.image.tag;
+  const graphqlStandardConfigRemoteBranch = deployGraphQLServerConfig.graphqlServer.standardConfigRemoteBranch;
 
   // File paths
   const filePaths = {
@@ -574,7 +603,7 @@ export async function configureFromSite(fromSite, toSite, env, deployUI, deployG
     await configureUI(SITE_HOST, CORE_URL, DOCUPACE_HOST, toSite, env, useNewDomain, auth0Domain, auth0ClientId, auth0ClientSecret, auth0Token, uiSecretsPath, uiImageTag, repoPath);
   }
 
-  await configureGraphQLServer(SITE_HOST, CORE_URL, DOCUPACE_HOST, toSite, env, auth0Domain, auth0ClientId, auth0ClientSecret, auth0Token, graphqlSecretsPath, graphqlImageTag, shortCommitId, repoPath, commonSecrets);
+  await configureGraphQLServer(SITE_HOST, CORE_URL, DOCUPACE_HOST, toSite, env, auth0Domain, auth0ClientId, auth0ClientSecret, auth0Token, graphqlSecretsPath, graphqlImageTag, graphqlStandardConfigRemoteBranch, repoPath, commonSecrets);
   await pushDeployConfig(repoPath, `Config update for ${toSite} from ${fromSite} with UI: ${uiImageTag} and GraphQL: ${graphqlImageTag}`);
 
   await initializeAuth0Configuration({
@@ -583,4 +612,11 @@ export async function configureFromSite(fromSite, toSite, env, deployUI, deployG
     auth0AuthToken: auth0Token,
     coreApiIdentifier: commonSecrets.CORE_API_IDENTIFIER,
   });
+
+  await addApplicationToApi({
+    application: 'docupace-graphql-client-credentials',
+    auth0Domain: auth0Domain,
+    auth0AuthToken: auth0Token,
+    coreApiIdentifier: commonSecrets.CORE_API_IDENTIFIER
+  })
 }

package/src/next/apache.js

@@ -0,0 +1,36 @@
+import { $ } from 'zx';
+
+export async function addApacheSite() {
+    if (
+      !process.env.ENVIRONMENT ||
+      (process.env.ENVIRONMENT !== 'dev' && process.env.ENVIRONMENT !== 'qa')
+    ) {
+      log("Error: ENVIRONMENT must be either 'dev' or 'qa'.");
+      process.exit(1);
+    }
+    if (!process.env.SITE) {
+      log('Error: SITE cannot be empty.');
+      process.exit(1);
+    }
+  
+    // Save private key from base64
+    const haproxyPrivateKey = process.env.HAPROXY_PRIVATE_KEY;
+    if (!haproxyPrivateKey) {
+      log('Error: HAPROXY_PRIVATE_KEY not set.');
+      process.exit(1);
+    }
+  
+    await fs.writeFile('private.key', Buffer.from(haproxyPrivateKey, 'base64'));
+    await fs.chmod('private.key', 0o600);
+  
+    const server = `${process.env.ENVIRONMENT}-apalb01a.corp.docupace.com`;
+    console.log("Running command");
+    const sshCmd = (script) =>
+      $`ssh -i private.key -o StrictHostKeyChecking=no root@${server} ${script}`;
+  
+    // The script is run twice in the Bash version
+    await sshCmd(`/root/repo/iis-preprod/scripts/addsite/add_core_site.sh ${process.env.ENVIRONMENT} ${process.env.SITE}`);
+    //await sshCmd(`/root/repo/iis-preprod/scripts/addsite/add_core_site.sh ${process.env.ENVIRONMENT} ${process.env.SITE}`);
+  }
+  
+  

package/src/next/cli.js

@@ -7,6 +7,10 @@ import { publish } from './publish.js';
 import { deployFromSite } from './deploy.js';
 import { deployTemplate } from './deploy.js';
 import { copy } from './copy.js';
+import { vaultAddSiteData } from './vault.js';
+import { addApacheSite } from './apache.js';
+import { haproxySync }  from './haproxy.js';
+import { connectClone } from './connect.js';
 import dotenv from 'dotenv';
 
 dotenv.config({ path: ['.env.pipeline.local', 'config/.env.pipeline.local'] });
@@ -17,6 +21,7 @@ const program = new Command();
 import { $ } from 'zx';
 import { deployCore } from './jenkins.js';
 import { runPipeline } from './pipline.js';
+
 const GIT_SHA = (await $`git log --pretty=format:'%h' -n 1`).stdout.trim();
 
 program
@@ -33,6 +38,39 @@ program
     await sonar(GIT_SHA);
   });
 
+program
+.command('addApacheSite')
+.description('Add site to apache')
+.action(async () => {
+  await addApacheSite();
+});
+
+program
+.command('connectClone')
+.description('Clone connect site')
+.action(async () => {
+  await connectClone();
+});
+
+
+program
+.command('haproxySync')
+.description('Sync haproxy config')
+.action(async () => {
+  await haproxySync();
+});
+
+
+
+program
+  .command('vaultAddSiteData')
+  .description('Add m2m auth0 data to vault')
+  .requiredOption('-s, --site-name <siteName>', 'Site name')
+  .action(async (options) => {
+    const { siteName,  } = options;
+    await vaultAddSiteData(siteName);
+  });
+
 program
   .command('publish')
   .description('Publish a Docker image')
@@ -83,7 +121,8 @@ program
     });
   });
 
-program
+
+  program
   .command('copy')
   .description('Copy a Docker image between registries')
   .requiredOption('-a, --application <application>', 'Application name')

package/src/next/connect.js

@@ -0,0 +1,167 @@
+import 'zx/globals'; // npm i zx
+import fs from 'fs-extra';
+import path from 'path';
+import axios from 'axios';
+import { $ } from 'zx';
+
+
+
+import {
+  cloneEnvRepository,
+  cloneRepository
+} from './deploy.js';
+
+
+import { getVaultSecrets } from '../auth0/vault.js';
+
+import {
+  getVaultSecretsConfigs,
+  environmentConfig
+} from './config.js';
+
+process.env.COMMON_VAULT_PATH = process.env.COMMON_VAULT_PATH || 'secret/data/common/config';
+process.env.VAULT_SKIP_VERIFY = process.env.VAULT_SKIP_VERIFY || 'true';
+process.env.CONNECT_DIR = process.env.CONNECT_DIR || 'connect';
+
+
+function log(msg) {
+    console.log(`[*] ${msg}`);
+  }
+  
+  /* --------------------------------
+   * 1. GIT CHECKOUT
+   * -------------------------------- */
+  async function gitCheckout(repository, destDir, branch = 'master') {
+    if (!(await fs.pathExists(destDir))) {
+      log(`Directory ${destDir} does not exist. Cloning repository...`);
+    } else {
+      log(`Directory ${destDir} exists. Removing it...`);
+      await fs.remove(destDir);
+      log(`Cloning ${destDir} repository...`);
+    }
+    
+    // Since we use zx, the environment variables can be read from process.env
+    // e.g. $BITBUCKET_USERNAME, $BITBUCKET_APP_PASSWORD
+    const username = process.env.BITBUCKET_USERNAME;
+    const password = process.env.BITBUCKET_APP_PASSWORD;
+  
+    // Construct clone command
+    const cloneUrl = `https://${username}:${password}@bitbucket.org/dp-cloud/${repository}.git`;
+  
+    // Perform the clone
+    await $`git clone --branch=${branch} --depth 1 ${cloneUrl} ${destDir}`;
+  }
+  
+  
+
+    const vaultToken = process.env.VAULT_TOKEN;
+    const vaultAddr = process.env.VAULT_ADDR;
+  
+
+export async function connectClone() {
+
+
+    if (!vaultToken || !vaultAddr) {
+      console.error("Error: VAULT_TOKEN and VAULT_ADDR must be set.");
+      throw new Error("VAULT_TOKEN and VAULT_ADDR must be set.");
+    }
+
+    const site = process.env.BUILD_SITE;
+    if (!site) {
+      log('Error: BUILD_SITE is not set');
+      process.exit(1);
+    }
+    const siteCommonVaultPath = `secret/data/${site}/common/config`;
+  
+  
+    // 2) getVaultSecrets
+    const secretsJson = await getVaultSecrets(siteCommonVaultPath);
+    if (!secretsJson.AUTH0_CLIENT_ID) {
+      log('Failed to retrieve secret data.');
+      process.exit(1);
+    }
+    const siteCommonSecrets = secretsJson;
+  
+    // 3) Export secrets as environment variables
+    process.env.AUTH0_CLIENT_ID = siteCommonSecrets.AUTH0_CLIENT_ID;
+    process.env.AUTH0_CLIENT_SECRET = siteCommonSecrets.AUTH0_CLIENT_SECRET;
+  
+    process.env.AUTH0_SUBDOMAIN="docupace-${BUILD_ENVIRONMENT}-${BUILD_SITE}"
+    process.env.AUTH0_REGION_DOMAIN="us.auth0.com"
+
+    log(`BUILD_ENVIRONMENT: ${process.env.BUILD_ENVIRONMENT}`);
+    log(`BUILD_SITE: ${process.env.BUILD_SITE}`);
+    log(`AUTH0_SUBDOMAIN: ${process.env.AUTH0_SUBDOMAIN}`);
+    log(`AUTH0_REGION_DOMAIN: ${process.env.AUTH0_REGION_DOMAIN}`);
+    log(`AUTH0_CLIENT_ID: ${process.env.AUTH0_CLIENT_ID}`);
+  
+    process.env.AUTH0_DOMAIN = `${process.env.AUTH0_SUBDOMAIN}.${process.env.AUTH0_REGION_DOMAIN}`;
+  
+    // 4) Clone 'connect-deploy' into CONNECT_DIR
+    await gitCheckout('connect-deploy', process.env.CONNECT_DIR);
+  
+    // 5) Create site clone (emulating the dock.sh call)
+    const cloneFrom = `${process.env.FROM_BUILD_ENVIRONMENT}/${process.env.FROM_BUILD_SITE}`;
+    const projectHome = path.resolve(process.env.CONNECT_DIR);
+  
+    const cloneFromPath = path.join(projectHome, 'envs', cloneFrom);
+    if (!(await fs.pathExists(cloneFromPath))) {
+      log('Clone template directory not found');
+      process.exit(1);
+    }
+  
+    const newSitePath = path.join(
+      projectHome,
+      'envs',
+      process.env.BUILD_ENVIRONMENT,
+      process.env.BUILD_SITE
+    );
+  
+    if (!(await fs.pathExists(newSitePath))) {
+      log('New site does not exist. Performing cloning...');
+      // This is where the bash script does: bash dock.sh clone ...
+      // We'll replicate with zx:
+      process.env.IMAGE_VERSION=$`cd ${cloneFromPath} && grep -E '^IMAGE_VERSION=' dock.sh | cut -f2 -d=`
+      await $`bash dock.sh clone ${process.env.BUILD_ENVIRONMENT} ${process.env.BUILD_SITE} ${process.env.IMAGE_VERSION}`;
+    } else {
+      log('Site already exists. Continuing Auth0 setup...');
+    }
+  
+    // 6) Retrieve Auth0 metadata
+    const targetSitePath = path.join(
+      projectHome,
+      'envs',
+      process.env.BUILD_ENVIRONMENT,
+      process.env.BUILD_SITE
+    );
+    await $`cd ${targetSitePath} && bash dock.sh getAuth0Metadata`;
+  
+    // 7) Check conf files, print logs
+    await $`grep "auth0" ${path.join(targetSitePath, 'conf/application.conf')}`;
+    await $`ls -all ${path.join(targetSitePath, 'conf')}`;
+    await $`ls -all ${path.join(targetSitePath, 'conf/saml')}`;
+    await $`cat ${path.join(targetSitePath, 'conf/saml/auth0.xml')}`;
+  
+    // 8) Git add, commit, push
+    await $`cd ${targetSitePath} && git checkout master && git pull --ff-only`;
+    await $`cd ${targetSitePath} && git remote -v`;
+    await $`cd ${targetSitePath} && git branch`;
+    await $`cd ${targetSitePath} && git status`;
+    await $`cd ${targetSitePath} && git add . ../sites.sh`;
+    await $`cd ${targetSitePath} && git status`;
+  
+    const numChanges = (
+      await $`cd ${targetSitePath} && git diff-index HEAD`
+    ).stdout.split('\n').filter(Boolean).length;
+  
+    if (numChanges > 0) {
+      const jiraTicket = process.env.JIRA_TICKET || 'NO_JIRA';
+      const message = `${jiraTicket} usdoc-s-builui job download_auth0_metadata for ${process.env.BUILD_ENVIRONMENT}/${process.env.BUILD_SITE}`;
+      log('About to commit changes to git');
+      await $`cd ${targetSitePath} && git commit -m ${message}`;
+      await $`cd ${targetSitePath} && git push -u origin master`;
+    } else {
+      log('No changes have been made. The site likely already has the correct configuration.');
+    }
+  }
+  

package/src/next/haproxy.js

@@ -0,0 +1,50 @@
+import { $ } from 'zx';
+
+
+export async function haproxySync() {
+    const repoPath = '/root/repo/docupace-ansible';
+    const inventory = path.join(repoPath, 'inventories/preprod/inventory');
+  
+    // Create haproxy.sh
+    const scriptContent = `
+  #!/bin/bash
+  
+  [[ ! -f ${inventory} ]] && ( echo "Alert: no inventory file: ${inventory}" ; exit 1 )
+  cd ${repoPath}
+  pwd
+  git stash
+  git pull origin master
+  git checkout master
+  git submodule update --remote
+  git submodule foreach --recursive git checkout master
+  git submodule foreach --recursive git pull origin master
+  
+  ansible-playbook -i inventories/preprod/inventory haproxy.yml -e "cluster=preprod"
+  
+  echo "proceeding"
+  echo "committing changes"
+  git add preprod-haproxy.cfg
+  git commit -m "Sync haproxy"
+  git push origin master
+  `;
+  
+    await fs.writeFile('haproxy.sh', scriptContent);
+  
+    // Private key
+    const haproxyPrivateKey = process.env.HAPROXY_PRIVATE_KEY;
+    if (!haproxyPrivateKey) {
+      log('Error: HAPROXY_PRIVATE_KEY not set.');
+      process.exit(1);
+    }
+  
+    await fs.writeFile('private.key', Buffer.from(haproxyPrivateKey, 'base64'));
+    await fs.chmod('private.key', 0o600);
+  
+    await $`cat private.key`; // Just to mimic the script's behavior
+  
+    // Copy script to remote
+    await $`scp -i private.key -o StrictHostKeyChecking=no haproxy.sh root@10.10.28.198:/tmp/haproxy.sh`;
+  
+    // Execute script on remote
+    await $`ssh -i private.key -o StrictHostKeyChecking=no root@10.10.28.198 "bash /tmp/haproxy.sh"`;
+  }

package/src/next/vault.js

@@ -0,0 +1,13 @@
+import { $ } from 'zx';
+
+
+export async function vaultAddSiteData(SITE) {
+    const AUTH0_CLIENT_ID = `${process.env.AUTH0_CLIENT_ID}`;
+    const AUTH0_CLIENT_SECRET = `${process.env.AUTH0_CLIENT_SECRET}`; 
+    await $`export VAULT_SKIP_VERIFY=true`;
+    await $`vault kv put secret/dp-${SITE}/common/config AUTH0_CLIENT_ID="${AUTH0_CLIENT_ID}" AUTH0_CLIENT_SECRET="${AUTH0_CLIENT_SECRET}"`
+}
+
+
+
+