@leverege/build-tools 2.48.5 → 2.49.0

package/src/docker-to-registry.sh

@@ -0,0 +1,611 @@
+#!/bin/bash
+if [ -z "$1" ]; then
+  echo "   Usage: docker-to-registry <version number>"
+  exit 1
+fi
+
+# printf "\n****LOCAL DOCKREATE****\n\n" && sleep 1
+
+# The base docker image maybe overridden with:
+#
+#   BASE_NODE_IMAGE="gallium-alpine" npm run dockerize <version>
+#
+BASE_NODE_IMAGE="${BASE_NODE_IMAGE:-iron-alpine}" # node 20, iron
+
+# Load common bash functions
+. `build-tools --bashfun`
+cat<<DEPRECATED_TOOL
+
+  `color y "***DEPRECATED docker-to-registry.sh - upgrade to docker-to-registry.mjs***"`
+
+DEPRECATED_TOOL
+sleep 3
+
+PKGTOP="$npm_config_local_prefix"
+cd $PKGTOP/docker
+
+if [ ! -x "$(command -v jq)" ];
+then
+ cat<<NO_JQ
+ 
+  `color r "***ERROR: Missing jq JSON processor"` => `color g 'brew install jq'`
+
+NO_JQ
+ exit 1
+fi
+
+if [ ! -d "$PKGTOP/dist" ];
+then
+  cat<<NO_DIST
+
+  `color r "***ERROR: Missing the dist directory, run this first => "``color g 'npm run build'`
+
+NO_DIST
+  exit 1
+fi
+
+function packageJson() {
+  result=$(jq -r $1 $PKGTOP/package.json)
+  if [ "$result" != "null" ]
+  then
+    echo $result
+  else
+    echo $2
+  fi
+}
+
+# Global Vars
+pluginFile="Dockerfile.plugin"
+
+# cleanup trapper for gracefully killing, which doesn't do much right now
+function cleanup() {
+  [ ! -z "$DOCKER_SKIP_CLEANUP" ] && printf "\nSkipping clean up\n\n" && exit 1
+  printf "\nCleaning up and exiting...\n\n"
+  rm -rf ./workspace ./scripts $SSHDIR .npmrc
+  exit 0
+}
+trap cleanup INT
+
+function strip_undefined() {
+  local result="$1"
+  [ "$1" == "undefined" ] && result=""
+  printf "$result"
+}
+
+function create_DockerfilePlugin() {
+  [ -f 'Dockerfile.plugin' ] && return
+
+  cat<<DOCKERPLUGIN > Dockerfile.plugin
+# Dockerfile.plugin - optionally extend the service Docker image
+#
+# This file may be used to run additional docker commands during the image
+# build process without needing to maintain a local custom Dockerfile. For
+# example, uncommenting the following docker RUN command will cause the
+# curl and vim packages to be added to the deployed image thus making them
+# available from the pod's command line on k8s:
+#
+# RUN apk update && apk add --no-cache curl vim
+#
+# Keep in mind that the Alpine Linux base image is used to keep image
+# footprints small, so adding packages "just because" is not considered
+# a best practice.
+DOCKERPLUGIN
+}
+
+# Useful links with information regarding k8s/docker/nodejs and PID 1
+#   No PID 1 for NodeJS     => https://bit.ly/2r4gHkZ
+#   What tini is all about  => https://bit.ly/2HsSxqP
+#   Graceful kills with k8s => https://bit.ly/2NONEJG
+#
+# Params: $1 = version from dockerize invocation
+#         $2 = additional packages to install (htop redis-cli etc)
+function create_Dockerfile() {
+  local regvers="${1}"
+  local apkadds="${2}"
+
+  create_DockerfilePlugin
+
+  echo "Creating the Dockerfile for node ${BASE_NODE_IMAGE}"
+  cat<<DOCKERBLD > Dockerfile
+# Version ${regvers} @ `date`
+#
+# The FROM directive sets the Base Image for subsequent instructions
+FROM node:${BASE_NODE_IMAGE} as intermediate
+ENV NODE_ENV production
+ 
+RUN mkdir -p /usr/src/app
+WORKDIR /usr/src/app
+
+# Install app dependencies
+COPY ./workspace/ /usr/src/app/
+ENV GRPC_VERBOSITY ERROR
+
+# --------------------------------------------------------------
+# copy the ssh keys into place, npm install, and remove them
+# --------------------------------------------------------------
+  
+# Install packages to install private repos with ssh keys
+COPY ./.npmrc /usr/src/app/.npmrc
+RUN apk --no-cache add openssh-client && \
+    apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
+      libstdc++ linux-headers make ${apkadds} && \
+    npm install -g npm@10 && \
+    npm ci --only=production --ignore-scripts --no-optional ${npmlogging} && \
+    rm -f /usr/src/app/.npmrc /root/.ssh/*
+ 
+# --------------------------------------------------------------
+# On to the real build now, the thing before was just an intermediate container
+# --------------------------------------------------------------
+ 
+FROM node:${BASE_NODE_IMAGE}
+ 
+# Install tini for PID 1 and replace shell with bash so we can source files
+RUN apk update && \
+    apk add --no-cache bash curl tini vim ${apkadds} && \
+    npm install -g npm@10 && \
+    rm /bin/sh && ln -s /bin/bash /bin/sh && \
+    mkdir -p /usr/src/app /tmp/levlog && \
+    chown node:node /usr/src/app
+DOCKERBLD
+
+  if [ -f "${pluginFile}" ];
+  then
+    printf "\n# ${pluginFile} BEGIN\n" >> Dockerfile
+    cat $pluginFile >> Dockerfile
+    printf "# ${pluginFile} END\n\n" >> Dockerfile
+  fi
+
+  cat<<EODOCKER >> Dockerfile
+ENTRYPOINT [ "/sbin/tini", "--" ]
+WORKDIR /usr/src/app
+ 
+COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
+
+USER ${runuser}
+COPY ./bashrc /home/node/.bashrc
+CMD [ "/bin/bash", "-c", "node index.js" ]
+EODOCKER
+} # end of create_Dockerfile
+
+# Params: $1 = version from dockerize invocation
+function create_bashrc() {
+  cat<<EOBASHRC > bashrc
+#!/bin/bash
+#
+# Version $1 @ `date`
+
+alias h=history
+
+alias ls='ls -CF --color=auto'
+alias ll='ls -lh'
+alias lla='ls -lha'
+alias glep='grep -l -s'
+alias m=less
+alias menv='env | sort | less'
+
+alias whatsmyip='wget -qO- ifconfig.co'
+
+alias err='wget -q -O- localhost:5111/logLevel/error'
+alias wrn='wget -q -O- localhost:5111/logLevel/warn'
+alias inf='wget -q -O- localhost:5111/logLevel/info'
+alias dbg='wget -q -O- localhost:5111/logLevel/debug'
+alias trc='wget -q -O- localhost:5111/logLevel/trace'
+
+socks()
+{
+  netstat -ant | awk '{print $6}' | sort | uniq -c | sort -n
+}
+
+cmetrics()
+{
+  wget -qO- localhost:5111/metrics
+}
+
+cmstat()
+{
+  wget -qO- localhost:5111\${1}
+}
+
+cmclear()
+{
+  cmstat /status/clear
+}
+EOBASHRC
+} # end of create_bashrc
+
+# Spilt a character delimited string and emit the object at the specified
+# index. So passing a full registry string in like:
+#   us-docker.pkg.dev/leverege-registry/leverege
+#
+# the project name may be extracted by calling this like:
+#   getNthOvjectFromDelimitedString $registry 1 '/'
+#
+function getNthObjectFromDelimitedString() {
+  local inputStr=$1
+  local index=$2
+  local delimeter=$3
+
+  IFS=$delimeter
+  read -ra ADDR <<< "$inputStr"
+
+  echo ${ADDR[$index]}
+}
+
+# ---------- MAIN PROCESSING STARTS HERE ----------
+#
+# make sure we have a current npmrc file
+refresh-npm-token
+
+# deprecation checks first...
+deprecated=$(packageJson '.leverege.project')
+if [ ! -z "$deprecated" ];
+then
+  cat<<DEPRECATED_PROJECT
+
+  $(color y '***DEPRECATED: leverege.project setting in package.json')
+
+  Setting the project in the package.json leverege section was used for builds
+  being stored in the deprecated GCP container registry. Remove the deprecated
+  leverege.project setting from package.json and try again.
+
+  "leverege": {
+    $(color r "\"project\": \"$deprecated\",")
+    ...
+  }
+
+DEPRECATED_PROJECT
+  exit 1
+fi
+
+deprecated=$(packageJson '.leverege.container')
+if [ ! -z "$deprecated" ];
+then
+  cat<<DEPRECATED_CONTAINER
+
+  $(color y '***DEPRECATED: leverege.container is no longer supported')
+
+  Setting the container name explicitly from the package.json leverge block
+  is no longer supported. The container name will be automatically derived
+  from the git repository's remote root, which is the default behavior. Remove
+  the container line from the leverege section in package.json and try again.
+
+  "leverege": {
+    $(color r "\"container\": \"$deprecated\",")
+    ...
+  }
+DEPRECATED_CONTAINER
+  exit 1
+fi
+
+deprecated=$(packageJson '.leverege.nodeimg')
+if [ ! -z "$deprecated" ];
+then
+  cat<<DEPRECATED_NODEIMAGE
+
+  $(color y '***DEPRECATED: leverege.nodeimg is no longer supported')
+
+  Setting the node base image using the nodeimg statement in package.json is no
+  longer supported. By default the actual node image version will be defaulted
+  by this script and wil rarely need to be a specific version.  Remove the
+  nodeimg line from the leverege section in package.json and try again.
+
+  "leverege": {
+    $(color r "\"nodeimg\": \"$deprecated\",")
+    ...
+  }
+DEPRECATED_NODEIMAGE
+  exit 1
+fi
+
+deprecated=$(packageJson '.leverege.artifact')
+if [ ! -z "$deprecated" ];
+then
+  cat<<DEPRECATED_ARTIFACT
+
+  $(color y '***DEPRECATED: leverege.artifact is no longer supported')
+
+  Setting the artifact registry folder is no longer supported. Instead use the
+  $(color g leverege.registry) setting to specify the full artifact registry and folder
+  used for storing the docker image. Remove the artifact line from the leverege
+  section in package.json and try again.
+
+  "leverege": {
+    $(color r "\"artifact\": \"$deprecated\",")
+    ...
+  }
+  
+  $(color g "New syntax example:")
+  
+  "leverege": {
+    $(color g "\"registry\": \"us-docker.pkg.dev/leverege-registry/$deprecated\",")
+  }
+DEPRECATED_ARTIFACT
+  exit 1
+fi
+
+deprecated=$(packageJson '.leverege.nodeops')
+if [ ! -z "$deprecated" ];
+then
+  cat<<DEPRECATED_NODEOPS
+
+  $(color y '***DEPRECATED: leverege.nodeops are no longer supported')
+
+  Setting the hard coded node options on the image is no longer supported. The
+  better approach is to add $(color g 'NODE_OPTIONS') to the config section of the chart's
+  values.yaml to allow downstream users to easily tune the options as needed.
+  Remove the nodeops line from the leverege section in package.json and try again.
+
+  "leverege": {
+    $(color r "\"nodeops\": \"$deprecated\",")
+    ...
+  }
+DEPRECATED_NODEOPS
+  exit 1
+fi
+
+# the container name will be the git root for the repo
+gitremote=`git config --get remote.origin.url`
+if [ -z $gitremote ];
+then
+  cat<<NOCONTAINER
+
+  $(color r '***ERROR: attempting to build an unnamed container')
+
+  You are attempting to create a docker image without a container name or
+  a git remote setup. Either define a $(color y 'container') statement in the leverege
+  section of package.json, or add a $(color g 'git remote') to this repository.
+
+NOCONTAINER
+  exit 1
+fi
+container=$(basename $gitremote .git)
+
+# build the registry specification
+registry=$(packageJson '.leverege.registry')
+if [ ! -z "$registry" ];
+then
+  gcp_project="$(getNthObjectFromDelimitedString $registry 1 '/')"
+  registry_folder=$registry/images/$container
+else
+  cat<<MISSING_REGISTRY
+
+  $(color r '***ERROR: the leverege.registry setting is required in package.json')
+
+  The $(color g 'leverege.registry') setting is now required and must be set to point to the
+  base URL of the artifact registry, plus the folder that the image is to be
+  stored in. For example, the Leverege Google project that contains our artifact
+  registry is $(color g 'leverege-registry') which contains several folders that are used to
+  group images by commonality. The stack services store their images in the
+  $(color g 'stack') folder, while general purpose services that we use may end up in the
+  $(color g 'leverege') folder.
+
+  Using api-server as an example, given this setting in api-server's package.json:
+
+  "leverege": {
+    $(color g '"registry": "us-docker.pkg.dev/leverege-registry/stack"')
+  }
+
+  the resultant docker image gets stored in:
+
+      $(color g 'us-docker.pkg.dev/leverege-registry/stack/images/api-server')
+
+  The helm/values.yaml file would then reference the same location:
+
+  image:
+    pullPolicy: IfNotPresent
+    registry: $(color g 'us-docker.pkg.dev/leverege-registry/leverege/images')
+    tag: ""
+MISSING_REGISTRY
+  exit 1
+fi
+
+# verify the cloud build bucket exists
+#
+build_bucket="gs://${gcp_project}_cloudbuild"
+gsutil ls -p $gcp_project -b $build_bucket &> /dev/null
+if [ "$?" -ne 0 ];
+then
+  cat<<MISSING_BUILD_BUCKET
+
+  $(color r "***ERROR: missing the cloud build bucket $build_bucket")
+
+  Go to Cloud Storage on the $(color g $gcp_project) GCP project and create the
+  target cloud build bucket before proceeding.
+
+  Build bucket => $(color g $build_bucket)
+MISSING_BUILD_BUCKET
+  exit 1
+fi
+
+version=$1
+[[ -f '.previous' ]] && previous=`cat .previous` || previous='FIRST BUILD'
+#pckgver="v`node -p \"require('../package.json').version\"`"
+pckgver="v$(packageJson '.version')"
+pckgdts="`date +\"%Y%m%d-%H%M\"`"
+# egrep was deprecated in 2007!
+tagging=`echo $version | egrep -e "^v\d+\.\d+\.\d+$"`
+
+if [ $tagging ];
+then
+  if [[ `git status --porcelain` ]]; then
+    cat<<DIRTAG
+
+  $(color r '***ERROR: attempting to tag a non-beta dirty git repository')
+
+  You are attempting to create a tagged release image for pushing to the
+  container registry, but there are locally modified files. This is not
+  allowed since the applied tag will not be relevant to the image version
+  due to the pending commits.
+
+  Available options for proceeding:
+
+    $(color g '1) cleanly commit all local mods assuming relevance')
+    $(color y '2) eliminate unwanted local modifications')
+    $(color r '3) stash anything that is irrelevant to this release')
+
+
+DIRTAG
+    exit 1
+  fi
+
+  # since we're tagging, the current branch must have an upstream
+  branch=`git rev-parse --abbrev-ref HEAD`
+  git rev-parse --abbrev-ref ${branch}@{u} &> /dev/null
+
+  if [ $? -ne 0 ];
+  then
+    cat<<NOUPSTR
+
+  $(color r '***ERROR: the current branch must have an upstream')
+
+  You are attempting to create a tagged release image for pushing to the
+  container registry, but the current branch does not have an upstream to
+  push to. This is not allowed since the applied tag will be stranded here
+  in your local repository, which is what we are attempting to avoid.
+
+  Available options for proceeding:
+
+    $(color g '1) set an upstream for this branch via:')
+    $(color g "       git push --set-upstream origin ${branch}")
+
+    $(color y '2) rebase, squash and merge onto a branch with an upstream')
+
+NOUPSTR
+    exit 1
+  fi
+
+  tagstat=$(color y 'will be tagged')
+else
+  tagstat=$(color r 'BETA RELEASE WILL NOT BE TAGGED')
+fi
+
+pluggedIn=$(color g "no")
+[ -f "${pluginFile}" ] && pluggedIn=$(color y "YES")
+
+apkadds=$(packageJson '.leverege.apkadds')
+runuser=$(packageJson '.leverege.runuser')
+[ -z "${runuser}" ] && runuser='node'
+
+npmlogging='--silent' # allow for verbose npm logs
+if [ "$VERBOSE_DOCKREATE_NPM" == "yes" ];
+then
+  npmlogging='--ddd'
+else
+  hintverbose=1
+fi
+
+echo "Updating dependencies and workspace..." && npm install
+if [ $? -ne 0 ];
+then
+  cat<<INSTALLFAILED
+
+  $(color y 'npm install') $(color r 'FAILED') - fix it in order to proceed
+
+INSTALLFAILED
+  exit 1
+fi
+
+cat<<IMGINFO
+  Project:   $(color g $gcp_project)
+  Registry:  $(color g $registry)
+  Container: $(color g $container)
+  Version:   $(color g $version)  $tagstat
+  Folder:    $(color y $registry_folder)
+  NodeImage: $(color g $BASE_NODE_IMAGE)
+  AddedPkgs: $(color g $apkadds)
+  PluggedIn: ${pluggedIn}
+  Run User:  $(color g $runuser)
+  Previous:  $(color y $previous)
+  DateStamp: $(color g $pckgdts)
+  NPM Logs:  $(color g $npmlogging)
+
+IMGINFO
+
+[ $hintverbose ] && printf "  Define $(color g VERBOSE_DOCKREATE_NPM=\"yes\") for verbose build logging\n\n"
+
+if [ "$pckgver" != "$version" ];
+then
+  cat<<EOPKG
+  $(color y '***WARNING: specified version does not match package.json version' )
+                   package.json => $(color g "$pckgver")
+                   specified    => $(color r "$version")
+
+EOPKG
+  if [ $tagging ];
+  then
+    cat<<EONOPE
+
+  $(color r '... and you are attempting to release so NOPE!' )
+
+EONOPE
+    exit 1
+  fi
+fi
+
+create_Dockerfile "$version" "$apkadds"
+create_bashrc "$version"
+
+rm -rf ./workspace && mkdir -p ./workspace
+cp -rfp ${PKGTOP}/package*json ${PKGTOP}/dist/* ./workspace
+ 
+# grab the npmrc token for the image
+cp -f "$HOME/.npmrc" .npmrc
+
+[ "$CIRCLECI" == "true" ] && exit 0
+
+printf "Hit return to continue or ^C to exit\n\n"; read ANS
+
+echo $version > .previous
+
+echo "Changing Google Project to $project"
+gcloud config set project $gcp_project
+
+echo "Creating Docker Image for $registry_folder:$version"
+echo "   Building the workspace"
+
+# use kaniko to cache builds
+if [ "$USE_KANIKO" == "yes" ];
+then
+  gcloud config set builds/use_kaniko True
+else
+  gcloud config unset builds/use_kaniko
+fi
+
+# need a logging area outside of the default cloud build bucket in order
+# to enable 3rd party builders from outside of leverege.com user space
+build_logs="--gcs-log-dir $build_bucket/logs"
+
+# submit the build request to gcloud with logs directed to the cloudbuild
+# logging bucket from the docker directory
+time gcloud builds submit $build_logs --tag $registry_folder:$version .
+ 
+if [ $? -ne 0 ];
+then
+  bldErr=$?
+  cat<<BUILDERR
+
+  $(color r '***ERROR: build failed! Aborting without tagging.')
+
+  The gcloud build process seems to have run into a problem. Please fix the
+  issue and rerun the dockerize process. This build will not be tagged, nor
+  will it be pushed.
+
+BUILDERR
+  exit $bldErr
+fi
+
+# Tag the container with the node detailsand the culprit.
+gcloud container images add-tag --quiet $registry_folder:$version \
+                                        $registry_folder:$pckgdts-$BASE_NODE_IMAGE-$USER \
+                                        $registry_folder:latest
+
+if [ $tagging ];
+then
+  printf "\n$(color g "Tagging and Pushing $version")\n"
+  git tag -af $1 -m "Docker version $1"
+  git push --follow-tags
+else
+  printf "\n$(color y '*** Skipped git tagging - beta release')\n\n"
+fi
+
+cleanup

package/src/helm-charts/api-server/values-local.yaml

@@ -4,6 +4,13 @@ image:
 config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
 
+# CNPG overrides
+# # stock pgsql
+# MODEL_SQL_HOST: "db-postgres-stack-rw.cnpg-operands"
+# PG_MODELS_HOST: "db-postgres-stack-rw.cnpg-operands"
+# # timescale
+# PG_HOST: "db-timescale-plain-ro.cnpg-operands"
+
 autoscaling:
   minReplicas: 3
   maxReplicas: 12

package/src/helm-charts/authz-server/values-local.yaml

@@ -5,3 +5,6 @@ config:
   LOG_CONFIG: '{"type":"pino","level":"warn"}'
   IN_MEM_CONTROLLER_ACTIVE: "false"
   USE_MEMORY_CONTROLLER_CHECK_MODE : "false"
+
+  # CNPG override
+# SQL_HOST: "db-postgres-stack-rw.cnpg-operands" # vs postgres-postgresql