@leverege/build-tools 2.65.0 → 2.65.1

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.65.0",
+  "version": "2.65.1",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/Utils.mjs

@@ -90,11 +90,12 @@ export const shellCmd = async ( cmdstr, opts = {} ) => {
   }
 }
 
-export const mkdirSafe = async ( dir ) => {
+export const mkdirSafe = ( dir ) => {
   try {
-    await fs.mkdir( dir, { recursive : true } ) // eslint-disable-line security/detect-non-literal-fs-filename
-  } catch ( e ) {
-    warning( `Could not create output dir: ${dir}` )
+    fs.mkdirSync( dir, { recursive : true } ) // eslint-disable-line security/detect-non-literal-fs-filename
+  } catch ( error ) {
+    condir( { error }, '<==fs.mkdir failed?' )
+    warning( { error }, `Could not create output dir: ${dir}` )
   }
 }
 

package/src/clone-cnpg-from-snapshot.mjs

@@ -25,8 +25,7 @@ program
   .requiredOption( '--source-project <project>', 'Source GCP project ID' )
   .requiredOption( '--dest-project <project>', 'Destination GCP project ID' )
   .requiredOption( '--dest-region <region>', 'Destination region, e.g. us-east4' )
-  .option( '--only-k8s-yaml', 'Only generate the volume snapshot YAML files' )
-  .option( '--emit-yaml-dir <dir>', 'Directory to emit VolumeSnapshot YAMLs', './out' )
+  .option( '--artifacts-only', 'Only generate the volume snapshot YAML, script and readme' )
   .option( '--dry-run', 'Show commands but don\'t run them' )
   .option( '--execute', 'Perform real actions (required to run gcloud commands)' )
   .option( '--cleanup', 'Remove temporary GCP resources after snapshot creation' )
@@ -42,7 +41,6 @@ const {
   destProject,
   destRegion,
   dryRun,
-  emitYamlDir,
   snapshot,
   sourceProject,
 } = options
@@ -55,6 +53,10 @@ const destSnapshot = `snapshot-${sourceProject}-${snapshot}`
 const tempDiskFromImage = `temp-from-img-${snapshot}`
 const destZone = `${destRegion}-a` // infer from region - implies "-a" zone always available
 
+// directory for receiving the YAML necessary for defining the snapshot class,
+// content and volume
+const emitYamlDir = `cnpg-clone-${snapshot}`
+
 let sourceZone // set and used in main(), also needed by cleanupResouces
 
 const getPrettyTimeNow = () => {
@@ -71,6 +73,61 @@ const getElapsedTime = ( startMs ) => {
   return `${minutes}:${seconds}`
 }
 
+// equivalent to Unix touch command to create an empty file
+const touchFile = async ( filename ) => {
+  if ( filename ) {
+    const handle = await fs.open( filename, 'w' ) // eslint-disable-line security/detect-non-literal-fs-filename
+    await handle.close()
+  }
+}
+
+// generates an executable script to use for defining the volume on k8s
+const writeApplyScript = async ( dir ) => {
+  const applyScript = `#!/ust/bin/env bash
+#
+# Apply VolumeSnapshot resources prior to deploying the CNPG cluster
+#
+kubectl apply -f ${dir}/volumesnapshotclass.yaml
+kubectl apply -f ${dir}/volumesnapshotcontent.yaml
+kubectl apply -f ${dir}/volumesnapshot.yaml
+`
+
+  const scriptPath = path.join( dir, 'apply-volumesnapshots.sh' )
+  await fs.writeFile( scriptPath, applyScript, { mode : 0o755 } ) // eslint-disable-line security/detect-non-literal-fs-filename
+  log( chalk.green( '  [✓] Helper script written:' ), scriptPath )
+}
+
+const writeReadme = async ( dir ) => {
+  const readmePath = path.join( dir, 'readme.txt' )
+  const content = `This directory contains the Kubernetes manifests needed to bootstrap a new CNPG cluster
+using a cloned volume snapshot from another GCP project.
+
+Files:
+  - volumesnapshotclass.yaml         (optional, use if the class doesn't already exist)
+  - volumesnapshotcontent.yaml       (refers to the GCE snapshot copied into this project)
+  - volumesnapshot.yaml              (used by CNPG to bootstrap from the volume snapshot)
+  - apply-volumesnapshots.sh         (helper script to apply these manifests in correct order)
+  - .nohelm                          (marker file for overwhelm to skip values.yaml generation)
+
+To deploy the volume snapshots to your Kubernetes cluster:
+  
+  $ ./apply-volumesnapshots.sh
+
+Once applied, reference the following VolumeSnapshot in your CNPG cluster bootstrap config:
+
+  spec.bootstrap.recovery.volumeSnapshot.volumeSnapshotName: ${k8sSnapshotName}
+
+Then deploy your CNPG cluster normally. Validate the cluster startup:
+
+  $ kubectl get pods -n cnpg-operands
+
+Enjoy your cloned cluster ✨
+`
+
+  await fs.writeFile( readmePath, content ) // eslint-disable-line security/detect-non-literal-fs-filename
+  log( chalk.green( '  [✓] README written:' ), readmePath )
+}
+
 const writeYaml = async ( filename, data ) => {
   const fullPath = path.join( emitYamlDir, filename )
   const yaml = YAML.dump( data )
@@ -98,7 +155,9 @@ const getVolumeHandle = async ( vscName ) => {
   return handle
 }
 
-const generateK8sYaml = async () => {
+// this function is responsible for emitting all of the artifacts for the creation
+// of the k8s volume that will then be used to bootstrap the cnpg cluster
+const emitCloningArtifacts = async () => {
   try {
     const snapshotYamlRaw = await shellCmd(
       `kubectl get volumesnapshot ${snapshot} -n cnpg-operands -o json`
@@ -120,6 +179,14 @@ const generateK8sYaml = async () => {
       )
     )
 
+    const snapshotClass = {
+      apiVersion : 'snapshot.storage.k8s.io/v1',
+      kind : 'VolumeSnapshotClass',
+      metadata : { name : 'cnpg-preprovisioned' },
+      driver : 'pd.csi.storage.gke.io',
+      deletionPolicy : 'Retain'
+    }
+
     const snapshotContent = {
       apiVersion : 'snapshot.storage.k8s.io/v1',
       kind : 'VolumeSnapshotContent',
@@ -155,16 +222,18 @@ const generateK8sYaml = async () => {
       },
     }
 
-    await mkdirSafe( emitYamlDir )
+    mkdirSafe( emitYamlDir )
+
+    await writeYaml( 'volumesnapshotclass.yaml', snapshotClass )
     await writeYaml( 'volumesnapshotcontent.yaml', snapshotContent )
     await writeYaml( 'volumesnapshot.yaml', snapshotYaml )
+    await touchFile( path.join( emitYamlDir, '.nohelm' ) )
+    await writeApplyScript( emitYamlDir )
+    await writeReadme( emitYamlDir )
+
   } catch ( err ) {
     errorExit( `Unable to generate VolumeSnapshot YAML: ${err.message}` )
   }
-
-  if ( options.onlyK8sYaml ) {
-    process.exit( 0 )
-  }
 }
 
 // runs a CLI command with a silly little spinner in an attempt to convince
@@ -246,7 +315,11 @@ const main = async () => {
   log( `        from handle: ${chalk.cyan( volumeHandle )}` )
   log( `            in zone: ${chalk.cyan( sourceZone )}\n` )
 
-  await generateK8sYaml()
+  // if all we want are the artifacts
+  if ( options.artifactsOnly ) {
+    await emitCloningArtifacts()
+    process.exit( 0 )
+  }
 
   log( `\n[STEP 2] 📸 Creating GCE snapshot from source disk [${diskName}]` )
   await runCommand( [
@@ -309,50 +382,59 @@ const main = async () => {
     `--project=${destProject}`,
   ] )
 
-  log( `\n✅ Done. Snapshot: ${destSnapshot} and recovery YAMLs ready.` )
+  await emitCloningArtifacts()
+
+  log( `
+  ✅ Done. Snapshot: ${chalk.cyan( destSnapshot )}
+           Recovery YAMLs written to: ${chalk.cyan( emitYamlDir )}
+           Run '${chalk.yellow( './apply-recovery-yaml.sh' )}' to prep the cluster for recovery.
+  ` )
 }
 
 const cleanupResources = async () => {
   log( '\n[STEP 9] 🧹 Cleaning up temporary resources (unwinding the stack)...' )
 
-  // STEP 7: Remove temp disk from image in destination project
-  await runCommand( [
-    `gcloud compute disks delete ${tempDiskFromImage}`,
-    `--zone=${destZone}`,
-    `--project=${destProject}`,
-    '--quiet'
-  ] )
-
-  // STEP 6: Remove copied image from destination project
-  await runCommand( [
-    `gcloud compute images delete ${tempImage}`,
-    `--project=${destProject}`,
-    '--quiet'
-  ] )
-
-  // STEP 4: Remove image from source project
-  await runCommand( [
-    `gcloud compute images delete ${tempImage}`,
-    `--project=${sourceProject}`,
-    '--quiet'
-  ] )
-
-  // STEP 3: Remove temp disk from source project
-  await runCommand( [
-    `gcloud compute disks delete ${tempDisk}`,
-    `--zone=${sourceZone}`,
-    `--project=${sourceProject}`,
-    '--quiet'
-  ] )
-
-  // STEP 2: Remove source snapshot
-  await runCommand( [
-    `gcloud compute snapshots delete ${intermediateSnapshot}`,
-    `--project=${sourceProject}`,
-    '--quiet'
-  ] )
+  const commands = [
+    [ // STEP 7: Remove temp disk from image in destination project
+      `gcloud compute disks delete ${tempDiskFromImage}`,
+      `--zone=${destZone}`,
+      `--project=${destProject}`,
+      '--quiet'
+    ],
+
+    [ // STEP 6: Remove copied image from destination project
+      `gcloud compute images delete ${tempImage}`,
+      `--project=${destProject}`,
+      '--quiet'
+    ],
+
+    [ // STEP 4: Remove image from source project
+      `gcloud compute images delete ${tempImage}`,
+      `--project=${sourceProject}`,
+      '--quiet'
+    ],
+
+    [ // STEP 3: Remove temp disk from source project
+      `gcloud compute disks delete ${tempDisk}`,
+      `--zone=${sourceZone}`,
+      `--project=${sourceProject}`,
+      '--quiet'
+    ],
+
+    [ // STEP 2: Remove source snapshot
+      `gcloud compute snapshots delete ${intermediateSnapshot}`,
+      `--project=${sourceProject}`,
+      '--quiet'
+    ],
+  ]
+
+  await Promise.all(
+    commands.map( cmdParts => runCommand( cmdParts ).catch( ( err ) => {
+      warning( `Non-fatal error during cleanup: ${err.message}` )
+    } ) )
+  )
 
-  log( chalk.green( '  [✓] Cleanup complete. Final GCE snapshot retained for CNPG bootstrapping.' ) )
+  log( chalk.green( '  [✓] Parallel complete. Final GCE snapshot retained for CNPG bootstrapping.' ) )
 }
 
 if ( cleanupOnly ) {

package/src/templates/dockerfileTemplatePy.hbs

@@ -9,13 +9,11 @@ RUN useradd -ms /bin/bash {{runuser}} && \
     mkdir -p /usr/src/app && \
     chown {{runuser}} /usr/src/app && \
     apt-get update && apt-get install -y \
-    wget git \
+    wget git libgl1 libglib2.0-0 \
     && rm -rf /var/lib/apt/lists/*
 
 #    gcc \
 #    git \
-#    libgl1 \
-#    libglib2.0-0 \
 #    && rm -rf /var/lib/apt/lists/*
 
 # Copy uv binary from the official distroless Docker image.