@leverege/build-tools 2.59.1 → 2.60.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.59.1",
+  "version": "2.60.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {

package/src/Docker.mjs

@@ -1,4 +1,6 @@
 import fs from 'node:fs'
+import path from 'node:path'
+import url from 'node:url'
 
 import chalk from 'chalk'
 import handlebars from 'handlebars'
@@ -13,6 +15,8 @@ import {
   warning
 } from './Utils.mjs'
 
+const DIRNAME = path.dirname( url.fileURLToPath( import.meta.url ) )
+
 // TODO: change default to node 22 (jod) after 01/01/25
 const dockerfileNodeVersion = 'jod-alpine'
 
@@ -20,74 +24,6 @@ const dockerfileNodeVersion = 'jod-alpine'
 // the build-tools repository, but it just became simpler to pull the contents
 // directly into these variables and forego the file loading.
 //
-const dockerfileTemplate = `
-# Version {{regvers}} @ {{date}}
-#
-# The FROM directive sets the Base Image for subsequent instructions
-FROM node:{{nodeimage}} as intermediate
-ENV NODE_ENV production
-ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
-
-RUN mkdir -p /usr/src/app
-WORKDIR /usr/src/app
-
-# Install app dependencies
-COPY ./workspace/ /usr/src/app/
-ENV GRPC_VERBOSITY ERROR
-
-# --------------------------------------------------------------
-# copy the ssh keys into place, npm install, and remove them
-# --------------------------------------------------------------
-
-# Install packages to install private repos with ssh keys
-COPY ./.npmrc \${NPM_CONFIG_USERCONFIG}
-RUN apk --no-cache add openssh-client && \
-    apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
-      libstdc++ linux-headers make {{apkadds}} && \
-    npm install -g {{npmVersion}}
-
-{{preInstallPluginfile}}
-
-# Use SKIP_PREPARE to disable hook-and-release from running
-ENV SKIP_PREPARE=true
-
-# Do the clean install and remove the npm token and ssh keys from the image
-RUN npm ci --only=production --no-optional {{npmlogging}} && \
-    rm -f \${NPM_CONFIG_USERCONFIG} /root/.ssh/*
-
-# --------------------------------------------------------------
-# Docker Final Stage
-# --------------------------------------------------------------
-
-FROM node:{{nodeimage}}
-ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
-
-# Copy .npmrc for private registry access for access to any @leverege packages
-COPY ./.npmrc \${NPM_CONFIG_USERCONFIG}
-
-# Install tini for PID 1 and replace shell with bash so we can source files
-RUN apk update && \
-    apk add --no-cache bash curl tini vim {{apkadds}} && \
-    npm install -g {{npmVersion}} && \
-    rm /bin/sh && ln -s /bin/bash /bin/sh && \
-    mkdir -p /usr/src/app /tmp/levlog && \
-    chown node:node /usr/src/app
-
-# Pull in the Dockerfile.plugin file here
-{{pluginfile}}
-
-# Eliminate the sensitive info from the final stage image
-RUN rm -f \${NPM_CONFIG_USERCONFIG}
-
-ENTRYPOINT [ "/sbin/tini", "--" ]
-WORKDIR /usr/src/app
-
-COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
-
-USER {{runuser}}
-COPY ./bashrc /home/node/.bashrc
-CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && node index.js" ]`
-
 const pluginTemplate = `
 # Dockerfile.plugin - optionally extend the service Docker image
 #
@@ -103,72 +39,15 @@ const pluginTemplate = `
 # footprints small, so adding packages "just because" is not considered
 # a best practice.
 `
-
-const bashrcTemplate = `#!/bin/bash
-#
-alias h=history
-
-alias ls='ls -CF --color=auto'
-alias ll='ls -lh'
-alias lla='ls -lha'
-alias glep='grep -l -s'
-alias m=less
-alias menv='env | sort | less'
-
-alias whatsmyip='wget -qO- ifconfig.co'
-
-alias err='wget -q -O- localhost:5111/logLevel/error'
-alias wrn='wget -q -O- localhost:5111/logLevel/warn'
-alias inf='wget -q -O- localhost:5111/logLevel/info'
-alias dbg='wget -q -O- localhost:5111/logLevel/debug'
-alias trc='wget -q -O- localhost:5111/logLevel/trace'
-
-socks()
-{
-  netstat -ant | awk '{print }' | sort | uniq -c | sort -n
-}
-
-cmetrics()
-{
-  wget -qO- localhost:5111/metrics
-}
-
-cmstat()
-{
-  wget -qO- localhost:5111
-}
-
-cmclear()
-{
-  # Future me - thinking this should call cmstat and pass in the endpoint as
-  # a parameter? Well don't do it - without escaping the dollar 1 in cmstat
-  # the port will end up being 51111, and escaping causes linting to complain
-  # about unnecessary escaping.
-  wget -qO- localhost:5111/status/clear
-}
-
-dunm() {
-  du -sh /usr/src/app/node_modules/* | sort -sh
-}
-`
-const cloudBuildSteps = `
-steps:
-  - name: 'gcr.io/cloud-builders/docker'
-    args:
-      - 'build' 
-      - '--network=cloudbuild'
-      - '-t'
-      - '{{imageName}}:{{imageVersion}}'
-      - '.'
-    
-images:
-  - '{{imageName}}:{{imageVersion}}'
-`
+const dockerfileTemplate = fs.readFileSync( path.join( DIRNAME, './templates/dockerfileTemplate.hbs' ), 'utf8' )
+const bashrcTemplate = fs.readFileSync( path.join( DIRNAME, './templates/bashrcTemplate.hbs' ), 'utf8' )
+const cloudBuildSteps = fs.readFileSync( path.join( DIRNAME, './templates/cloudBuildSteps.hbs' ), 'utf8' )
 
 const defaultSettings = {
   apkadds : '',
   date : getDateTimestamp(),
   nodeimage : 'jod-alpine',
+  imageBase : 'node:jod-alpine',
   npmlogging : process.env.VERBOSE_NPM_LOGGING ? '--ddd' : '--silent',
   npmVersion : 'npm@11',
   pluginfile : '# NO PLUGIN',
@@ -196,7 +75,7 @@ const generateDockerfile = ( options ) => {
   if ( !settings.nodeimage ) {
     settings.nodeimage = dockerfileNodeVersion
   }
-
+  settings.image = settings.imageBase || `node:${settings.nodeimage}`
   // Ensure the presence of the docker dir with plugin and bashrc - this code
   // could probably be tightened up a little.
   try {
@@ -207,7 +86,9 @@ const generateDockerfile = ( options ) => {
   }
 
   const bashrcFile = './docker/bashrc'
-  if ( !fs.existsSync( bashrcFile ) ) {
+  // DISABLING exists check. With addition of Bashrc.plugin, the file needs to
+  // always be created or we encounter errors
+  if ( true ) { // !fs.existsSync( bashrcFile ) ) {
     let bf = bashrcTemplate
     const bashrcFilePlugin = './docker/Bashrc.plugin'
     if ( fs.existsSync( bashrcFilePlugin ) ) {

package/src/circleate.sh

@@ -120,6 +120,7 @@ jobs:
           path: reports
 
   docs:
+    circleci_ip_ranges: true
     docker:
       - image: $CIRCLE_NODE_IMG
     steps:
@@ -339,6 +340,7 @@ jobs:
           path: reports
 
   docs:
+    circleci_ip_ranges: true
     docker:
       - image: $CIRCLE_NODE_IMG
     steps:

package/src/docker-to-registry.mjs

@@ -96,6 +96,8 @@ if ( semver.lt( semver.coerce( repoDescr.buildToolsVersion ), minimumBuildToolsV
 
 const dockerInfo = await docker.generateDockerfile( {
   regvers : repoDescr.packageVersion,
+  imageBase : repoDescr.closestPackageJson.leverege?.imageBase,
+  buildEnv : repoDescr.closestPackageJson.leverege?.buildEnv || 'alpine',
   nodeimage : repoDescr.closestPackageJson.leverege?.nodeimg, // left as nodeimg in package.json for backwards compat
 } )
 debug( { dockerInfo }, '<==The Docker Info' )
@@ -216,7 +218,8 @@ log( `
     NPM Workspace: ${chalk.green.bold( isNpmWorkspace )}
 
   ${chalk.green.bold( 'Docker Information:' )}
-    NodeImage:     ${chalk.green.bold( dockerInfo.nodeimage )}
+    NodeImage:     ${chalk.green.bold( dockerInfo.imageBase || `node:${dockerInfo.nodeimage}` )}
+    Docker Env:    ${chalk.green.bold( dockerInfo.buildEnv || 'alpine' )}
     AddedPkgs:     ${chalk.green.bold( dockerInfo.apkadds )}
     Run User:      ${chalk.green.bold( dockerInfo.runuser )}
     Previous:      ${chalk.yellow.bold( dockerInfo.previousBuild )}

package/src/templates/bashrcTemplate.hbs

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+alias h=history
+
+alias ls='ls -CF --color=auto'
+alias ll='ls -lh'
+alias lla='ls -lha'
+alias glep='grep -l -s'
+alias m=less
+alias menv='env | sort | less'
+
+alias whatsmyip='wget -qO- ifconfig.co'
+
+alias err='wget -q -O- localhost:5111/logLevel/error'
+alias wrn='wget -q -O- localhost:5111/logLevel/warn'
+alias inf='wget -q -O- localhost:5111/logLevel/info'
+alias dbg='wget -q -O- localhost:5111/logLevel/debug'
+alias trc='wget -q -O- localhost:5111/logLevel/trace'
+
+socks()
+{
+  netstat -ant | awk '{print }' | sort | uniq -c | sort -n
+}
+
+cmetrics()
+{
+  wget -qO- localhost:5111/metrics
+}
+
+cmstat()
+{
+  wget -qO- localhost:5111
+}
+
+cmclear()
+{
+  # Future me - thinking this should call cmstat and pass in the endpoint as
+  # a parameter? Well don't do it - without escaping the dollar 1 in cmstat
+  # the port will end up being 51111, and escaping causes linting to complain
+  # about unnecessary escaping.
+  wget -qO- localhost:5111/status/clear
+}
+
+dunm() {
+  du -sh /usr/src/app/node_modules/* | sort -sh
+}

package/src/templates/cloudBuildSteps.hbs

@@ -0,0 +1,13 @@
+steps:
+  - name: 'gcr.io/cloud-builders/docker'
+    args:
+      - 'build' 
+      - '--network=cloudbuild'
+      - '-t'
+      - '{{imageName}}:{{imageVersion}}'
+      - '--cache-from'
+      - '{{imageName}}:{{imageVersion}}'
+      - '.'
+    
+images:
+  - '{{imageName}}:{{imageVersion}}'

package/src/templates/dockerfileTemplate.hbs

@@ -0,0 +1,86 @@
+# Version {{regvers}} @ {{date}}
+
+# The FROM directive sets the Base Image for subsequent instructions
+FROM {{image}} as intermediate
+
+ARG BUILD_ENV="{{buildEnv}}"
+
+ENV NODE_ENV production
+ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
+
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY ./workspace/ /usr/src/app/
+ENV GRPC_VERBOSITY ERROR
+
+# --------------------------------------------------------------
+# copy the ssh keys into place, npm install, and remove them
+# --------------------------------------------------------------
+
+# Install packages to install private repos with ssh keys
+COPY ./.npmrc ${NPM_CONFIG_USERCONFIG}
+
+RUN if [ "${BUILD_ENV}" = "debian" ]; then \
+      apt-get update && \
+      apt-get install -y wget openssh-client build-essential && \
+      npm install -g npm@10; \
+    else \
+      apk --no-cache add openssh-client && \
+      apk --update add --no-cache --virtual build-dep g++ gcc libgcc libstdc++ linux-headers make {{apkadds}} && \
+      npm install -g npm@10; \
+    fi
+
+# Use SKIP_PREPARE to disable hook-and-release from running
+ENV SKIP_PREPARE=true
+
+{{preInstallPluginfile}}
+
+# Do the clean install and remove the npm token and ssh keys from the image
+RUN npm ci --only=production --no-optional {{npmlogging}} && \
+    rm -f ${NPM_CONFIG_USERCONFIG} /root/.ssh/*
+
+# --------------------------------------------------------------
+# Docker Final Stage
+# --------------------------------------------------------------
+
+FROM {{image}}
+
+ARG BUILD_ENV="{{buildEnv}}"
+
+ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
+
+# Copy .npmrc for private registry access for access to any @leverege packages
+COPY ./.npmrc ${NPM_CONFIG_USERCONFIG}
+
+# Install tini for PID 1 and replace shell with bash so we can source files
+RUN if [ "$BUILD_ENV" = "debian" ]; then \
+      apt-get update && \
+      apt-get install -y tini bash curl vim {{apkadds}} && \
+      npm install -g npm@10 && \
+      rm /bin/sh && ln -s /bin/bash /bin/sh && \
+      mkdir -p /usr/src/app /tmp/levlog && chown node:node /usr/src/app; \
+    else \
+      apk update && \
+      apk add --no-cache bash curl tini vim {{apkadds}} && \
+      npm install -g npm@10 && \
+      rm /bin/sh && ln -s /bin/bash /bin/sh && \
+      mkdir -p /usr/src/app /tmp/levlog && chown node:node /usr/src/app; \
+    fi
+
+
+# Pull in the Dockerfile.plugin file here
+{{pluginfile}}
+
+# Eliminate the sensitive info from the final stage image
+RUN rm -f ${NPM_CONFIG_USERCONFIG}
+
+ENTRYPOINT [ "tini", "--" ]
+WORKDIR /usr/src/app
+
+COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
+
+USER {{runuser}}
+COPY ./bashrc /home/node/.bashrc
+CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && node index.js" ]

package/src/build-cnpg-image.mjs

@@ -1,167 +0,0 @@
-#!/usr/bin/env node
-
-// A simple utility for easily building CNPG based psql images. The resultant
-// image will contain PostgreSQL, the JS query and Timescale extensions, and
-// the extensions that are required by CNPG like barman, pgaudit, pgvector and
-// pg-failover-slots.
-//
-// Latest Versions:
-//   postgres    => https://www.postgresql.org/
-//   timescale   => https://docs.timescale.com/about/latest/
-//   CNPG images => https://github.com/cloudnative-pg/postgres-containers/pkgs/container/postgresql
-
-import fs from 'node:fs'
-import path from 'node:path'
-
-import chalk from 'chalk'
-import cliArgs from 'command-line-args'
-import cliHelp from 'command-line-usage'
-import Handlebars from 'handlebars'
-
-import {
-  debug,
-  log,
-  err,
-  errorExit,
-  shellCmd,
-} from './Utils.mjs'
-
-// Define CLI options
-const optionList = [
-  { name : 'sub-tag', type : String, description : '{green sub-tag to add to the image tag (required)}', defaultOption : true },
-  { name : 'pg-version', type : String, description : '{green pgsql version (default: 16.6)}' },
-  { name : 'version', type : Boolean, description : '{green show script version}' },
-  { name : 'help', type : Boolean, description : '{green display this help screen}' },
-]
-
-const sections = [
-  {
-    header : 'Leverege CNPG Docker Image Builder',
-    content : `{green   This tool will build a PostgreSQL based image that is suitable for
-deployment by the CNPG operator. It builds the docker image by using an
-official CNPG docker image as a base and added the jsquery and timescale
-extensions for use within a Leverege k8s cluster. The resultant image may be
-used in the standard PostgreSQL configuration which is used by the stack
-components, as well as timescale based services like transponder tsdb and
-dense history.}`
-  },
-  {
-    header : 'Options',
-    optionList
-  },
-  {
-    header : 'Leverege Registry Image Tag',
-    content : `{green The full registry image tag that is used for tagging the docker image in the artifact registry}`,
-  },
-]
-const args = cliArgs( optionList, { partial : true } )
-const help = cliHelp( sections )
-
-debug( { args, help }, '<==Command Line Info?' )
-
-// Show help if needed
-if ( args.help ) {
-  console.log( help )
-  process.exit( 0 )
-}
-
-// Define defaults
-const PG_VERSION = args.pgVersion || '16.4'
-const { subTag } = args
-if ( !subTag ) {
-  errorExit( `Error: An image version must be specified.` )
-}
-
-const LEVEREGE_REGISTRY = 'us-docker.pkg.dev/leverege-registry/system/images/leverege-pgsql'
-const LEVEREGE_IMAGE_TAG = `${PG_VERSION}-cnpg-lvrg-${subTag}`
-const LEVEREGE_IMAGE_NAME_WITH_TAG = `${LEVEREGE_REGISTRY}:${LEVEREGE_IMAGE_TAG}`
-
-const DOCKERFILE_TEMPLATE = `
-FROM ghcr.io/cloudnative-pg/postgresql:{{PG_VERSION}}-bookworm
-
-USER root
-
-RUN set -xe; \\
-    apt-get update; \\
-    apt-get install -y --no-install-recommends \\
-        "postgresql-\${PG_MAJOR}-jsquery"; \\
-    rm -fr /tmp/*; \\
-    rm -rf /var/lib/apt/lists/*;
-
-RUN apt-get update \\
-    && apt-get install -y lsb-release wget \\
-    && echo "deb https://packagecloud.io/timescale/timescaledb/debian/ \$(lsb_release -c -s) main" | tee /etc/apt/sources.list.d/timescaledb.list \\
-    && wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | apt-key add - \\
-    && apt-get update \\
-    && apt-get install -y "timescaledb-2-postgresql-\${PG_MAJOR}" \\
-    && apt-get remove -y lsb-release wget \\
-    && rm -fr /tmp/* \\
-    && rm -rf /var/lib/apt/lists/*;
-
-RUN usermod -u 26 postgres
-USER 26
-`
-
-// Check for existing image
-const checkExistingImage = async ( imageName ) => {
-  const dockerResults= await shellCmd( `docker image list ${imageName} --quiet` )
-  debug( { imageName, dockerResults }, '<==docker image list' )
-  return dockerResults !== ''
-}
-
-// Build and push Docker image
-const buildAndPushImage = async () => {
-  const template = Handlebars.compile( DOCKERFILE_TEMPLATE )
-  const renderedDockerfile = template( { PG_VERSION } )
-  debug( { renderedDockerfile }, '<==Dockerfile' )
-
-  // Write the Dockerfile to a temporary location
-  const dockerfilePath = path.resolve( 'Dockerfile.temp' )
-  fs.writeFileSync( dockerfilePath, renderedDockerfile )
-
-  // Build the image
-  console.log( chalk.yellow( 'Building Docker image...' ) )
-  await shellCmd( `docker build --file ${dockerfilePath} --tag ${LEVEREGE_IMAGE_NAME_WITH_TAG} .` )
-
-  // Push the image
-  console.log( chalk.green( 'Pushing Docker image...' ) )
-  await shellCmd( `docker push ${LEVEREGE_IMAGE_NAME_WITH_TAG}` )
-
-  // Clean up the temporary Dockerfile
-  fs.unlinkSync( dockerfilePath )
-}
-
-// Main execution flow
-try {
-  if ( await checkExistingImage( LEVEREGE_IMAGE_NAME_WITH_TAG ) ) {
-    console.error(
-      `${chalk.red( '*** ERROR ***' ) 
-        } The image already exists:\n` +
-      `  Image   : ${chalk.yellow( LEVEREGE_IMAGE_NAME_WITH_TAG )}\n` +
-      `  Registry: ${chalk.green( LEVEREGE_REGISTRY )}\n` +
-      `  Tag     : ${chalk.green( LEVEREGE_IMAGE_TAG )}\n`
-    )
-    process.exit( 1 )
-  }
-
-  console.log( `${chalk.cyan( 'About to build a new Docker image:\n' ) 
-    }  PostgreSQL   => ${chalk.green( PG_VERSION )}\n` +
-    `  Image Name   => ${chalk.green( LEVEREGE_IMAGE_NAME_WITH_TAG )}\n` )
-
-  console.log( `Enter "${chalk.green( 'yes' )}" to proceed or anything else to exit.` )
-  const confirmation = await new Promise( ( resolve ) => {
-    process.stdin.once( 'data', data => resolve( data.toString().trim() ) )
-  } )
-
-  if ( confirmation !== 'yes' ) {
-    console.log( chalk.red( 'Aborted by user.' ) )
-    process.exit( 1 )
-  }
-
-  await buildAndPushImage()
-  console.log( chalk.green( 'Docker image build and push completed successfully!' ) )
-  process.exit( 0 )
-} catch ( error ) {
-  console.error( chalk.red( 'An error occurred:' ), error.message )
-  process.exit( 1 )
-}