@leverege/build-tools 2.57.4 → 2.58.0-pedro.2

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.57.4",
+  "version": "2.58.0-pedro.2",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -48,6 +48,7 @@
     "pkglint": "src/pkglint.sh",
     "prune-git": "src/prune-git.sh",
     "pull-git": "src/pull-git.sh",
+    "push-model-to-registry": "src/push-model-to-registry.mjs",
     "push-my-chart": "src/push-my-chart.mjs",
     "refresh-npm-token": "src/refresh-npm-token.mjs",
     "refresh-py-idx": "src/refresh-py-idx.sh",
@@ -59,6 +60,7 @@
   "license": "SEE LICENSE IN LICENSE.md",
   "dependencies": {
     "@google-cloud/artifact-registry": "^3.5.0",
+    "@google-cloud/storage": "^7.14.0",
     "ansi-colors": "^4.1.3",
     "chalk": "^5.3.0",
     "command-line-args": "^6.0.1",
@@ -66,6 +68,7 @@
     "deepmerge": "^4.3.1",
     "enquirer": "^2.4.1",
     "execa": "^9.5.1",
+    "fast-crc32c": "^2.0.0",
     "glob": "^11.0.0",
     "handlebars": "^4.7.8",
     "inquirer": "^12.1.0",
@@ -78,10 +81,10 @@
     "readline-sync": "^1.4.10",
     "semver": "^7.6.3",
     "simple-git": "^3.27.0",
-    "zx": "^8.2.2"
+    "zx": "^8.2.4"
   },
   "devDependencies": {
     "@leverege/eslint-config-leverege": "^5.0.1",
-    "npm": "^10.9.0"
+    "npm": "^10.9.1"
   }
-}
+}

package/src/Docker.mjs

@@ -23,6 +23,7 @@ const dockerfileTemplate = `
 # The FROM directive sets the Base Image for subsequent instructions
 FROM node:{{nodeimage}} as intermediate
 ENV NODE_ENV production
+ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
 
 RUN mkdir -p /usr/src/app
 WORKDIR /usr/src/app
@@ -36,7 +37,7 @@ ENV GRPC_VERBOSITY ERROR
 # --------------------------------------------------------------
 
 # Install packages to install private repos with ssh keys
-COPY ./.npmrc /usr/src/app/.npmrc
+COPY ./.npmrc \${NPM_CONFIG_USERCONFIG}
 RUN apk --no-cache add openssh-client && \
     apk --update add --no-cache --virtual build-dep g++ gcc libgcc \\
       libstdc++ linux-headers make {{apkadds}} && \
@@ -44,16 +45,22 @@ RUN apk --no-cache add openssh-client && \
 
 {{preInstallPluginfile}}
 
-# Removed --ignore-scripts from npm ci because it was causing issues with python native build
+# Use SKIP_PREPARE to disable hook-and-release from running
 ENV SKIP_PREPARE=true
+
+# Do the clean install and remove the npm token and ssh keys from the image
 RUN npm ci --only=production --no-optional {{npmlogging}} && \
-    rm -f /usr/src/app/.npmrc /root/.ssh/*
+    rm -f \${NPM_CONFIG_USERCONFIG} /root/.ssh/*
 
 # --------------------------------------------------------------
-# On to the real build now, the thing before was just an intermediate container
+# Docker Final Stage
 # --------------------------------------------------------------
 
 FROM node:{{nodeimage}}
+ENV NPM_CONFIG_USERCONFIG /usr/src/app/.npmrc
+
+# Copy .npmrc for private registry access for access to any @leverege packages
+COPY ./.npmrc \${NPM_CONFIG_USERCONFIG}
 
 # Install tini for PID 1 and replace shell with bash so we can source files
 RUN apk update && \
@@ -63,8 +70,12 @@ RUN apk update && \
     mkdir -p /usr/src/app /tmp/levlog && \
     chown node:node /usr/src/app
 
+# Pull in the Dockerfile.plugin file here
 {{pluginfile}}
 
+# Eliminate the sensitive info from the final stage image
+RUN rm -f \${NPM_CONFIG_USERCONFIG}
+
 ENTRYPOINT [ "/sbin/tini", "--" ]
 WORKDIR /usr/src/app
 

package/src/circleate.sh

@@ -11,6 +11,7 @@ fi
 
 repo_type=$1
 CIRCLE_NODE_IMG="cimg/node:lts"
+DOC_CREATOR_ENDPOINT="https://doc-creator.leverege.com/buildDocs/$(basename $npm_package_name)"
 
 check_circle_config(){
   if [ -d "./.circleci" ];
@@ -131,6 +132,9 @@ jobs:
       - store_artifacts:
           path: docs
           prefix: docs
+      - run:
+          name: Initiate document creation
+          command: curl -X GET "$DOC_CREATOR_ENDPOINT"
 
   coverage:
     # coverage runs tests, generates coverage, and saves docs
@@ -347,6 +351,9 @@ jobs:
       - store_artifacts:
           path: docs
           prefix: docs
+      - run:
+          name: Initiate document creation
+          command: curl -X GET "$DOC_CREATOR_ENDPOINT"
 
   coverage:
     # coverage runs tests, generates coverage, and saves docs

package/src/helm-charts/prom-operator/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "The Prometheus Operator and Components"
 addHelmRepo prometheus-community https://prometheus-community.github.io/helm-charts
 
 showInstalling "The Prometheus Operator (kube-prometheus-stack)"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="65"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="66"
 
 NS="--namespace prometheus"
 

package/src/helm-charts/prom-operator/prometheus-stack.yaml.ovh

@@ -33,6 +33,8 @@ defaultRules:
 
 prometheus:
   prometheusSpec:
+#   image:
+#     tag: "v3.0.0"
     podMonitorSelectorNilUsesHelmValues: false
     ruleSelectorNilUsesHelmValues: false
     serviceMonitorSelectorNilUsesHelmValues: false
@@ -55,6 +57,11 @@ grafana:
   # -- the grafana admin password
   adminPassword: leverege-monitoring
   defaultDashboardsEnabled: false
+  persistence:
+    enabled: true
+    size: 10Gi
+    accessModes:
+      - ReadWriteOnce
   sidecar:
       dashboards:
         enabled: true
@@ -68,7 +75,7 @@ grafana:
         entryPoints:
           - websecure
         routes:
-          - match: Host(`OVH:<PROJECT_NAME>-promop.OVH:<HOST>.com`)
+          - match: Host(`${PROJECT_NAME}-promop.${HOST}.com`)
             kind: Rule
             services:
             - name: prometheus-stack-grafana
@@ -98,10 +105,10 @@ alertmanager:
     receivers:
       - name: slack
         slack_configs:
-          - channel: "OVH:<SLACK_CHANNEL>"
-            api_url: "OVH:<SLACK_HOOK_URL>"
-            username: "OVH:<PROJECT_NAME>"
-            fallback: "OVH:<PROJECT_NAME> - {{ .CommonAnnotations.summary }}"
+          - channel: "${SLACK_CHANNEL}"
+            api_url: "${SLACK_HOOK_URL}"
+            username: "${PROJECT_NAME}"
+            fallback: "${PROJECT_NAME} - {{ .CommonAnnotations.summary }}"
             title: "{{ range .Alerts }}{{ .Annotations.summary }}\n{{ end }}"
             title_link: "https://bitbucket.org/leverege/sre-playbook/src/master/"
             send_resolved: true
@@ -114,9 +121,9 @@ alertmanager:
       - name: slack-watchdog
         slack_configs:
           - channel: "#dead-mans-snitch_notifications"
-            api_url: "OVH:<SLACK_HOOK_URL>"
-            username: "OVH:<PROJECT_NAME>"
-            fallback: "OVH:<PROJECT_NAME> - {{ .CommonAnnotations.summary }}"
+            api_url: "${SLACK_HOOK_URL}"
+            username: "${PROJECT_NAME}"
+            fallback: "${PROJECT_NAME} - {{ .CommonAnnotations.summary }}"
             title: "Watchdog Alert - Dead Man's Snitch"
             # text: "This is a periodic test alert to confirm the alert pipeline is functional."
             text: |-
@@ -127,7 +134,7 @@ alertmanager:
               {{ end }}
       - name: opsgenie
         opsgenie_configs:
-          - api_key: "OVH:<OPSGENIE_APIKEY>"
+          - api_key: "${OPSGENIE_APIKEY}"
             priority: "{{ .CommonLabels.priority }}"
             tags: "{{ range .Alerts }}{{ .Labels.severity }},{{ end }}"
             description: |-

package/src/helm-charts/prom-operator/stackdriver-exporter.yaml.ovh

@@ -5,7 +5,7 @@
 fullnameOverride: "prometheus-stack-stackdriver-metrics"
 
 stackdriver:
-  projectId: "OVH:<PROJECT_NAME>"
+  projectId: "${PROJECT_NAME}"
   metrics:
     typePrefixes: "pubsub.googleapis.com/subscription/oldest_unacked_message_age,pubsub.googleapis.com/subscription/num_undelivered_messages,pubsub.googleapis.com/subscription/pull_ack_request_count,pubsub.googleapis.com/subscription/streaming_pull_ack_request_count,pubsub.googleapis.com/topic/message_sizes,pubsub.googleapis.com/topic/send_request_count,firebasedatabase.googleapis.com/io/database_load,firebasedatabase.googleapis.com/network/sent_bytes_count,firebasedatabase.googleapis.com/network/active_connections"
 

package/src/helm-charts/prom-operator/update-crd.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+VERSION="v0.78.1"
+CRDOPTS="--force-conflicts --server-side"
+
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagerconfigs.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_alertmanagers.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_podmonitors.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_probes.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_prometheusagents.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_prometheuses.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_prometheusrules.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_scrapeconfigs.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_servicemonitors.yaml
+kubectl apply $CRDOPTS -f https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/$VERSION/example/prometheus-operator-crd/monitoring.coreos.com_thanosrulers.yaml

package/src/helm-charts/redis/helmup.plugin

@@ -4,7 +4,7 @@ showInstalling "Redis"
 
 OCI_CHART="oci://registry-1.docker.io/bitnamicharts/redis"
 
-[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.2.1"
+[ -z "$REDIS_CHART_VERSION" ] && REDIS_CHART_VERSION="20.3.0"
 
 helm upgrade --install redis $OCI_CHART       \
              --values  redis/redis-local.yaml \

package/src/overwhelm.mjs

@@ -15,20 +15,29 @@ import parse from 'parse-gitignore'
 import ask from 'readline-sync'
 import YAML from 'js-yaml'
 
-import { debug, disableDebug, err, errorExit, isString, log, shellCmd } from './Utils.mjs'
+import {
+  debug,
+  disableDebug,
+  err,
+  errorExit,
+  isString,
+  log,
+  shellCmd,
+} from './Utils.mjs'
 
 const optionsDefinitions = [
   /* eslint-disable no-multi-spaces */
-  { name : 'context',  alias : 'x', type : String, default : '' },
-  { name : 'section',  alias : 's', type : String, default : 'default' },
-  { name : 'key',      alias : 'k', type : String },
-  { name : 'export',   alias : 'e', type : Boolean, default : false },
-  { name : 'continue', alias : 'c', type : Boolean, default : false },
-  { name : 'checkctx', alias : 't', type : Boolean, default : false }, // checks overwhelm context
-  { name : 'fetchctx', alias : 'f', type : Boolean, default : false }, // fetch and set k8s context
-  { name : 'genvals',  alias : 'g', type : Boolean, default : false }, // only generates values.yaml
-  { name : 'matchk8s', alias : 'm', type : Boolean, default : false }, // checks k8s context
-  { name : 'verbose',  alias : 'v', type : Boolean, default : false },
+  { name : 'context',    alias : 'x', type : String, default : '' },
+  { name : 'deprecated', alias : 'd', type : Boolean, default : false },
+  { name : 'section',    alias : 's', type : String, default : 'default' },
+  { name : 'key',        alias : 'k', type : String },
+  { name : 'export',     alias : 'e', type : Boolean, default : false },
+  { name : 'continue',   alias : 'c', type : Boolean, default : false },
+  { name : 'checkctx',   alias : 't', type : Boolean, default : false }, // checks overwhelm context
+  { name : 'fetchctx',   alias : 'f', type : Boolean, default : false }, // fetch and set k8s context
+  { name : 'genvals',    alias : 'g', type : Boolean, default : false }, // only generates values.yaml
+  { name : 'matchk8s',   alias : 'm', type : Boolean, default : false }, // checks k8s context
+  { name : 'verbose',    alias : 'v', type : Boolean, default : false },
   /* eslint-enable no-multi-spaces */
 ]
 const args = cliArgs( optionsDefinitions )
@@ -36,6 +45,12 @@ const args = cliArgs( optionsDefinitions )
 // when using --key we only want the key contents w/o debug info
 if ( args.key ) { disableDebug() }
 
+// define a deprecation warning function that depends on arg.deprecated
+const deprecated = ( deprecation ) => {
+  if ( !args.deprecated ) { return }
+  log( `${chalk.yellow.bold( '***DEPRECATED:' )} ${deprecation}\n` )
+}
+
 // See if we are in a monorepo, which changes some of the rules.
 const gitTop = await shellCmd( 'git rev-parse --show-toplevel' )
 
@@ -180,7 +195,7 @@ const yams = glob.sync( 'values-*.yaml' )
 
 // A function to apply the replacements to all of the matching yaml which
 // will error out if it runs into an unknown OVH<tag>.
-const doReplacements = ( yamls, replacements ) => {
+const doReplacements = ( yamls, replacements, fileName = 'default-config.yaml' ) => {
   let replYaml = yamls
 
   // wrap special chars with quotes if needed, like "#---incident-alerts---"
@@ -188,10 +203,18 @@ const doReplacements = ( yamls, replacements ) => {
     return /[#$?*]/g.test( raw ) ? `"${raw}"` : raw
   }
 
-  Object.keys( replacements ).forEach( ( repl ) => {
-    const replaceThis = new RegExp( `OVH:<${repl}>`, 'g' ) // eslint-disable-line security/detect-non-literal-regexp
-    const withThat = quoteSpecials( `${replacements[repl]}` )
-    replYaml = replYaml.replace( replaceThis, withThat )
+  // process both ${VARIABLE} and deprecate OVH:<VARIABLES> styles
+  Object.keys( replacements ).forEach( ( key ) => {
+    // replace ${VARIABLE} syntax
+    const replaceVariable = new RegExp( `\\$\\{${key}\\}`, 'g' ) // eslint-disable-line security/detect-non-literal-regexp
+    replYaml = replYaml.replace( replaceVariable, quoteSpecials( `${replacements[key]}` ) )
+
+    // replace OVH:<VARIABLE> syntax and warn about deprecation
+    const replaceDeprecated = new RegExp( `OVH:<${key}>`, 'g' ) // eslint-disable-line security/detect-non-literal-regexp
+    if ( replYaml.match( replaceDeprecated ) ) {
+      deprecated( `OVH:<${key}> is deprecated. Please replace it with \${${key}} in ${fileName}` )
+      replYaml = replYaml.replace( replaceDeprecated, quoteSpecials( `${replacements[key]}` ) )
+    }
   } )
 
   // Verify all replaceables have been handled, or error out.
@@ -296,7 +319,6 @@ ahoy.forEach( ( dir ) => {
     const valuesLocal = `${dir}/values-local.yaml`
 
     if ( validValuesLocalConfig( valuesLocal ) ) {
-      debug( { valuesLocal }, 'rendering values.yaml' )
       valuesOut += '\n# OVERWHELM => values-local.yaml\n\n'
       valuesOut += fs.readFileSync( valuesLocal, { encoding : 'utf-8' } )
     }
@@ -308,28 +330,17 @@ ahoy.forEach( ( dir ) => {
     const valuesAll = YAML.load( valuesOut ) || {}
     const valuesSvc = {
       service : {
-        project_id : 'OVH:<PROJECT_ID>',
-        project_name : 'OVH:<PROJECT_NAME>',
-        host : 'OVH:<HOST>',
-        ingress : 'OVH:<INGRESS>',
+        /* eslint-disable no-template-curly-in-string */
+        project_id : '${PROJECT_ID}',
+        project_name : '${PROJECT_NAME}',
+        host : '${HOST}',
+        ingress : '${INGRESS}',
+        /* eslint-enable no-template-curly-in-string */
       }
     }
     const valuesMerged = YAML.dump( deepmerge( valuesSvc, valuesAll ), { quotingType : '"' } )
 
-    valuesOut = doReplacements( valuesMerged, over )
-
-    // DEPRECATED once siren and manheim are platform 4.0
-    // If we are going to override then we need to convert it to YAML and
-    // do a deepmerge to combine the values.
-    const valuesOverride = `${dir}/values-override.yaml`
-
-    if ( fs.existsSync( valuesOverride ) ) {
-      log( `===>Applying Overrides=> ${valuesOverride}` )
-      const overrides = YAML.load( fs.readFileSync( valuesOverride, { encofind : 'utf-8' } ) )
-      const allValues = YAML.load( valuesOut )
-      valuesOut = YAML.dump( deepmerge( allValues, overrides ), { quotingType : '"' } )
-    }
-    // DEPRECATED
+    valuesOut = doReplacements( valuesMerged, over, valuesLocal )
 
     fs.writeFileSync( `${dir}/values.yaml`, valuesOut )
   }
@@ -338,8 +349,9 @@ ahoy.forEach( ( dir ) => {
 // we can also apply replaceables to all .ovh files in the project
 glob.sync( '**/*.ovh' ).forEach( ( cfgmap ) => {
   const target = cfgmap.replace( /.ovh$/, '' )
+  deprecated( `rename deprecated ${cfgmap} OVH file to ${target}` )
   let valuesOut = fs.readFileSync( cfgmap, { encoding : 'utf-8' } )
-  valuesOut = doReplacements( valuesOut, over )
+  valuesOut = doReplacements( valuesOut, over, cfgmap )
   fs.writeFileSync( target, valuesOut )
 } )
 

package/src/push-model-registry.mjs

@@ -0,0 +1,145 @@
+#!/usr/bin/env node
+
+import fs from 'node:fs';
+import path from 'node:path';
+import chalk from 'chalk';
+import inquirer from 'inquirer';
+import {Storage} from '@google-cloud/storage';
+import crc32c from 'fast-crc32c';
+
+const storage = new Storage();
+
+function getAllFiles(dirPath, arrayOfFiles = []) {
+  const files = fs.readdirSync(dirPath);
+
+  files.forEach(file => {
+    const filePath = path.join(dirPath, file);
+    if (fs.lstatSync(filePath).isDirectory()) {
+      arrayOfFiles = getAllFiles(filePath, arrayOfFiles);
+    } else {
+      arrayOfFiles.push(filePath);
+    }
+  });
+
+  return arrayOfFiles;
+}
+
+async function main() {
+  // Get git branch name
+  const branchName = fs.readFileSync('.git/HEAD', 'utf8').split('/').pop().trim();
+
+  const bucketName = `${branchName}-triton-model-registry`;
+
+  // Get all blobs in the bucket
+  let files = [];
+  try {
+    [files] = await storage.bucket(bucketName).getFiles();
+  } catch (error) {
+    if (error.code === 404) {
+      console.error(`Bucket ${bucketName} does not exist.`);
+      return;
+    } else {
+      throw error;
+    }
+  }
+
+  // Get all not hidden blobs in the current path in the filesystem
+  const localFiles = getAllFiles('./models').filter(file => !path.basename(file).startsWith('.'));
+
+  const filesToDelete = files.filter(file => !localFiles.find(localFile => localFile === file.name));
+
+  console.log(`${chalk.underline(filesToDelete.length + " file(s)")} in ${chalk.underline(bucketName)} without a ${chalk.underline("local")} counterpart`);
+  filesToDelete.forEach(file => console.log(chalk.red(file.name)));
+
+  if (filesToDelete.length > 0) console.log("");
+
+  const filesToAdd = [];
+  const filesToReplace = [];
+
+  const metadataPromises = localFiles.map(async file => {
+    const localFileChecksum = Buffer.from(crc32c.calculate(fs.readFileSync(file)).toString(16), 'hex').toString('base64');
+    const remoteFile = files.find(blob => blob.name === file);
+    if (remoteFile) {
+      const [metadata] = await remoteFile.getMetadata();
+      return { file, localFileChecksum, remoteFileChecksum: metadata.crc32c };
+    } else {
+      return { file, localFileChecksum, remoteFileChecksum: null };
+    }
+  });
+
+  const metadataResults = await Promise.all(metadataPromises);
+
+  metadataResults.forEach(({ file, localFileChecksum, remoteFileChecksum }) => {
+    if (remoteFileChecksum) {
+      if (localFileChecksum !== remoteFileChecksum) {
+        filesToReplace.push(file);
+      }
+    } else {
+      filesToAdd.push(file);
+    }
+  });
+
+  console.log(`${chalk.underline(filesToAdd.length + " file(s)")} to be added to ${chalk.underline(bucketName)}`);
+  filesToAdd.forEach(file => console.log(chalk.green(file)));
+
+  if (filesToAdd.length > 0) console.log("");
+
+  console.log(`${chalk.underline(filesToReplace.length + " file(s)")} to be replaced in ${chalk.underline(bucketName)}`);
+  filesToReplace.forEach(file => console.log(chalk.yellow(file)));
+
+  if (filesToReplace.length > 0) console.log("");
+
+  if (filesToDelete.length > 0) {
+    const {confirm} = await inquirer.prompt({
+      type: 'confirm',
+      name: 'confirm',
+      message: `Do you want to delete ${filesToDelete.length} file(s) from ${bucketName}?`,
+    });
+
+    if (!confirm) {
+      return;
+    }
+
+    // Remove all blobs that are not in the local filesystem
+    const deletePromises = filesToDelete.map(async file => {
+      console.log(`Deleting ${file.name} from ${bucketName}`);
+      return storage.bucket(bucketName).file(file.name).delete();
+    });
+    
+    await Promise.all(deletePromises);
+    console.log("")
+  }
+
+  if (filesToAdd.length > 0 || filesToReplace.length > 0) {
+    const {confirm} = await inquirer.prompt({
+      type: 'confirm',
+      name: 'confirm',
+      message: `Do you want to upload ${filesToAdd.length + filesToReplace.length} file(s) to ${bucketName}?`,
+    });
+
+    if (!confirm) {
+      return;
+    }
+
+    // Upload all new files
+    const uploadPromises = filesToAdd.map(async file => {
+      console.log(`Uploading ${file} to ${bucketName}`);
+      return storage.bucket(bucketName).upload(file, {
+        destination: file,
+      });
+    });
+
+    // Replace all files with different checksums
+    const replacePromises = filesToReplace.map(async file => {
+      console.log(`Replacing ${file} in ${bucketName}`);
+      return storage.bucket(bucketName).upload(file, {
+        destination: file,
+      });
+    });
+    
+    await Promise.all([...uploadPromises, ...replacePromises]);
+    console.log("")
+  }
+}
+
+main().catch(console.error);