@leverege/build-tools 2.115.2 → 2.117.0

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leverege/build-tools",
-  "version": "2.115.2",
+  "version": "2.117.0",
   "description": "A collection of build / support tools for Leverege developers",
   "main": "index.js",
   "repository": {
@@ -23,7 +23,6 @@
     "build-tools": "src/build-tools.mjs",
     "chart-compass": "src/chart-compass.mjs",
     "chart-to-registry": "src/chart-to-registry.mjs",
-    "circle-orb-it": "src/circle-orb-it/circle-orb-it.mjs",
     "clone-cnpg-from-snapshot": "src/clone-cnpg-from-snapshot.mjs",
     "count-lines-of-code": "src/count-lines-of-code.mjs",
     "dashboard": "src/dashboard/dashboard.mjs",

package/src/Docker.mjs

@@ -148,6 +148,10 @@ const generateDockerfile = async ( options ) => {
   if ( fs.existsSync( dockerfilePrePlugin ) ) {
     settings.preInstallPluginfile = fs.readFileSync( dockerfilePrePlugin )
   }
+  const dockerfileFinalStagePrePlugin = './docker/DockerfileFinalStagePreInstall.plugin'
+  if ( fs.existsSync( dockerfileFinalStagePrePlugin ) ) {
+    settings.finalStagePreInstallPluginfile = fs.readFileSync( dockerfileFinalStagePrePlugin )
+  }
 
   const compiled = handlebars.compile( dockerfileTemplate, { noEscape : true } )
   const replaced = compiled( settings )

package/src/Utils.mjs

@@ -338,14 +338,14 @@ export const parseJsonFile = async ( jsonFile ) => {
   }
 }
 
-// Parses the ./package.json file and verifies there is a properly formatted
-// leverege.registry section present.
 const deprecated = ( opts ) => {
   log( chalk.red.bold( `\n  ***DEPRECATED: ${opts.deprecationError}` ) )
   log( chalk.white( opts.deprecationInfo ) )
   process.exit( 1 )
 }
 
+// Parses the ./package.json file and verifies there is a properly formatted
+// leverege.registry section present.
 export const parsePackageJson = async ( packageFileName = './package.json' ) => {
   if ( !fs.existsSync( packageFileName ) ) { return undefined }  
   

package/src/bash-funcs

@@ -55,6 +55,15 @@ function clearlyWarn() {
 CLEARWARN
 }
 
+function btDeprecated() {
+  cat<<BT_DEPRECATED
+
+  `color r "*** DEPRECATED: ${1}"`
+
+BT_DEPRECATED
+  sleep 3
+}
+
 function warning() {
   clearlyWarn DO_NOT_CLEAR
 }

package/src/chart-compass.mjs

@@ -113,6 +113,23 @@ const buildCompassMap = ( yamlFile ) => {
   }
 
   const chartCompass = YAML.load( fs.readFileSync( yamlFile, 'utf8' ) ) // eslint-disable-line security/detect-non-literal-fs-filename
+
+  if ( !Array.isArray( chartCompass ) ) {
+    errorExit( `${yamlFile}: expected a YAML array at the top level` )
+  }
+
+  for ( const entry of chartCompass ) {
+    const repositories = entry.repositories || {}
+    for ( const [ repo, chartMap ] of Object.entries( repositories ) ) {
+      if ( chartMap === null || typeof chartMap !== 'object' || Array.isArray( chartMap ) ) {
+        errorExit(
+          `${yamlFile}: repository "${repo}" must be a map of chart entries (each chart key needs ": {}" or explicit config).\n` +
+          `  Got: ${JSON.stringify( chartMap )?.slice( 0, 120 )}`
+        )
+      }
+    }
+  }
+
   return chartCompass
 }
 

package/src/circle-orb-it/circle-orb-it.mjs

@@ -57,6 +57,8 @@ if ( options.version ) {
   process.exit( 0 )
 }
 
+warning( 'circle-orb-it is deprecated. Use github-actions-it instead.' )
+
 const gitRoot = await getGitRootDirectory()
 if ( !gitRoot ) {
   errorExit( 'This command must be run inside a git repository' )

package/src/custom-images/kubectl/build-kubetcl.sh

@@ -2,7 +2,7 @@
 #
 # builds an alpine image with kubectl installed
 #
-KUBECTL_VERSION="1.30.3"
+KUBECTL_VERSION="1.31.0"
 
 docker buildx build \
   --build-arg KUBECTL_VERSION=${KUBECTL_VERSION} \

package/src/docker-to-registry.mjs

@@ -129,7 +129,7 @@ const dockerfileOptions = {
   isNpmWorkspace,
   buildEnv : leveregeStanza.buildEnv || leveregeStanza['build-env'] || 'alpine',
   imageBase : leveregeStanza.imageBase || leveregeStanza['image-base'],
-  runuser : leveregeStanza.runUser || leveregeStanza['run-user'] || isNodeJsProject ? 'node' : 'python',
+  runuser : leveregeStanza.runUser || leveregeStanza['run-user'] || ( isNodeJsProject ? 'node' : 'python' ),
   nodeimage : leveregeStanza.nodeimg, // left as nodeimg in package.json for backwards compat
   regvers : repoDescr.packageVersion,
   pythonVersion : repoDescr.rootPyProjectToml?.pythonVersion,

package/src/gh-audit-log.mjs

@@ -0,0 +1,155 @@
+#!/usr/bin/env node
+import { $ } from 'zx'
+import { program } from 'commander'
+import chalk from 'chalk'
+
+import { enableDebug, errorExit, log } from './Utils.mjs'
+
+$.verbose = false
+
+const DEFAULT_ORG = 'Leverege'
+const DEFAULT_LIMIT = 50
+
+program
+  .name( 'gh-audit-log' )
+  .description( chalk.green( 'Query the Leverege GitHub org audit log' ) )
+  .option( '--org <org>', 'GitHub org', DEFAULT_ORG )
+  .option( '--team <slug>', 'Filter to a team (e.g. workwatch-admins) — applied client-side, GitHub has no server-side team qualifier' )
+  .option( '--repo <name>', 'Filter to a repo (e.g. tpi-viz) — server-side' )
+  .option( '--actor <login>', 'Filter to a specific actor' )
+  .option( '--action <phrase>', 'Filter by action prefix (e.g. team, repo, org)' )
+  .option( '--since <date>', 'Only show events after this date (YYYY-MM-DD)' )
+  .option( '--phrase <raw>', 'Raw audit-log search phrase, combined with the above' )
+  .option( '--limit <n>', 'Max events to show', DEFAULT_LIMIT )
+  .option( '--json', 'Print raw JSON instead of a formatted table' )
+  .option( '--debug', 'Enable debug logging' )
+  .addHelpText( 'after', `
+${chalk.yellow( 'Examples:' )}
+  ${chalk.green( 'gh-audit-log --team workwatch-admins' )}
+  ${chalk.green( 'gh-audit-log --actor leverege-jphelps --since 2026-08-14' )}
+  ${chalk.green( 'gh-audit-log --action repo --since 2026-08-01 --json' )}
+
+${chalk.yellow( 'Notes:' )}
+  Requires org-owner access — this hits ${chalk.green( 'GET /orgs/{org}/audit-log' )}, same
+  endpoint used throughout the GitHub access-control work this tool grew out of.
+
+  GitHub's audit-log search has no server-side "team" qualifier, even though events carry
+  a team field — ${chalk.green( '--team' )} is matched client-side after fetching.
+
+  This tool is built for access-control auditing, not as a general org-activity browser.
+  Unless you pass --action / --phrase / --repo / --actor, --action defaults to 'team' —
+  otherwise an unscoped query pages through the entire org log, which is large even over
+  a few days (routine org activity plus a chatty self-hosted-runner bot that registers
+  itself hourly). --since also defaults to 7 days ago unless given explicitly. Use
+  --action org (or --phrase) to look outside team/permission events, e.g. org role changes.
+` )
+  .parse()
+
+const opts = program.opts()
+if ( opts.debug ) { enableDebug(); $.verbose = true }
+
+// ---------- Build the audit-log search phrase (server-side qualifiers only) ----------
+
+// This tool exists for access-control auditing, not as a general org-activity browser —
+// default to action:team (team/permission changes) unless the caller opts into something
+// broader via --action/--phrase/--repo/--actor. Without this, an unscoped query pages
+// through the entire org log, which is large even over a few days (routine org activity
+// plus a chatty self-hosted-runner bot that registers hourly).
+const hasExplicitScope = opts.action || opts.phrase || opts.repo || opts.actor
+const action = opts.action ?? ( !opts.phrase && !hasExplicitScope ? 'team' : undefined )
+const since = opts.since ?? sevenDaysAgo()
+
+const phraseParts = []
+if ( opts.repo ) { phraseParts.push( `repo:${opts.org}/${opts.repo}` ) }
+if ( opts.actor ) { phraseParts.push( `actor:${opts.actor}` ) }
+if ( action ) { phraseParts.push( `action:${action}` ) }
+if ( since ) { phraseParts.push( `created:>${since}` ) }
+if ( opts.phrase ) { phraseParts.push( opts.phrase ) }
+const phrase = phraseParts.join( ' ' )
+
+function sevenDaysAgo() {
+  const d = new Date()
+  d.setDate( d.getDate() - 7 )
+  return d.toISOString().slice( 0, 10 )
+}
+
+// ---------- Fetch ----------
+
+async function fetchAuditLog( { org, phrase : searchPhrase } ) {
+  const phraseArgs = searchPhrase ? [ '-f', `phrase=${searchPhrase}` ] : []
+
+  let stdout
+  try {
+    ( { stdout } = await $`gh api orgs/${org}/audit-log -X GET --paginate ${phraseArgs}` )
+  } catch ( error ) {
+    errorExit( `Failed to query audit log: ${error.stderr ?? error.message ?? error}` )
+  }
+
+  // `gh api --paginate` concatenates one JSON array per page back-to-back —
+  // split and re-flatten rather than assuming a single JSON document.
+  const pages = stdout
+    .split( /(?<=\])\s*(?=\[)/ )
+    .filter( chunk => chunk.trim().length > 0 )
+    .map( chunk => JSON.parse( chunk ) )
+
+  return pages.flat()
+}
+
+const fetched = await fetchAuditLog( { org : opts.org, phrase } )
+const events = opts.team ?
+  fetched.filter( e => e.team === `${opts.org}/${opts.team}` ) :
+  fetched
+const limited = events.slice( 0, Number( opts.limit ) )
+
+if ( opts.json ) {
+  log( JSON.stringify( limited, null, 2 ) )
+  process.exit( 0 )
+}
+
+if ( limited.length === 0 ) {
+  log( chalk.yellow( 'No matching audit log events found.' ) )
+  process.exit( 0 )
+}
+
+// ---------- Formatted table ----------
+
+const rows = limited.map( e => ( {
+  when : new Date( e['@timestamp'] ).toLocaleString(),
+  actor : e.actor ?? '-',
+  action : e.action ?? '-',
+  team : e.team ? e.team.replace( `${opts.org}/`, '' ) : '-',
+  repo : e.repo ? e.repo.replace( `${opts.org}/`, '' ) : '-',
+  permission : e.permission ?? '-',
+} ) )
+
+const widths = [ 'when', 'actor', 'action', 'team', 'repo', 'permission' ].reduce( ( acc, key ) => {
+  acc[key] = Math.max( key.length, ...rows.map( r => String( r[key] ).length ) )
+  return acc
+}, {} )
+
+const PERMISSION_COLORS = {
+  admin : chalk.red,
+  maintain : chalk.yellow,
+  write : chalk.green,
+  triage : chalk.gray,
+  read : chalk.gray,
+}
+
+const printRow = ( row, { header = false } = {} ) => {
+  const cells = Object.entries( widths ).map( ( [ key, width ] ) => {
+    // pad the plain text first — ANSI color codes have zero visible width but
+    // count toward .length, so padding *after* colorizing would misalign columns
+    const padded = String( row[key] ).padEnd( width )
+    if ( header ) { return chalk.bold( padded ) }
+    if ( key === 'permission' ) { return ( PERMISSION_COLORS[row[key]] ?? chalk.dim )( padded ) }
+    if ( key === 'actor' ) { return row[key].endsWith( '[bot]' ) ? chalk.dim( padded ) : padded }
+    if ( key === 'when' ) { return chalk.dim( padded ) }
+    return padded
+  } )
+  log( cells.join( '  ' ) )
+}
+
+printRow( { when : 'WHEN', actor : 'ACTOR', action : 'ACTION', team : 'TEAM', repo : 'REPO', permission : 'PERM' }, { header : true } )
+rows.forEach( row => printRow( row ) )
+
+log( chalk.dim( `\n${limited.length} of ${events.length} matching events shown (--limit ${opts.limit} to change)` ) )

package/src/github-actions-it.mjs

@@ -18,12 +18,24 @@ import {
 } from './Utils.mjs'
 
 const GH_ACTIONS_REF = 'development'
-const CIRCLE_YML = '.circleci/config.yml'
 const GH_WORKFLOWS_DIR = '.github/workflows'
 const GH_CI_YML = `${GH_WORKFLOWS_DIR}/ci.yml`
+const GH_DEPENDABOT_YML = '.github/dependabot.yml'
 const DEFAULT_COVERAGE_THRESHOLD = 70
 const DEFAULT_CACHE_CONFIG = '{"type":"inMemory"}'
 
+const DEPENDABOT_CONTENT = `version: 2
+updates:
+  - package-ecosystem: "npm"
+    directory: "/"
+    schedule:
+      interval: "weekly"
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: "eslint"
+        versions: [">=10"]
+`
+
 program
   .name( 'github-actions-it' )
   .description( chalk.green( 'Installs or updates GitHub Actions CI configuration for a Leverege library repository' ) )
@@ -31,11 +43,12 @@ program
   .option( '--dry-run', 'Show what would be written without making changes' )
   .addHelpText( 'after', `
 ${chalk.yellow( 'Files managed:' )}
-  ${chalk.green( GH_CI_YML )}     GitHub Actions CI workflow
+  ${chalk.green( GH_CI_YML )}           GitHub Actions CI workflow
+  ${chalk.green( GH_DEPENDABOT_YML )}  weekly npm dependency updates
 
 ${chalk.yellow( 'Notes:' )}
   Run from anywhere inside a git repository.
-  Reads .circleci/config.yml to preserve existing settings when present.
+  Preserves settings from an existing ci.yml when updating.
   Redis is always available on port 6379 — no configuration needed.
   Yarn auth and nodeLinker are handled by the workflow writing ~/.yarnrc.yml on the runner.
 
@@ -53,60 +66,11 @@ if ( !gitRoot ) {
   errorExit( 'This command must be run inside a git repository' )
 }
 
-// ---------- Parse existing CircleCI config ----------
-
-const circleYmlPath = path.join( gitRoot, CIRCLE_YML )
+// ---------- Defaults ----------
 
 let enableGcp = false
 let coverageThreshold = DEFAULT_COVERAGE_THRESHOLD
 let cacheConfig = DEFAULT_CACHE_CONFIG
-let redisEnabled = false
-let isYarnFromCircle = false
-
-if ( fs.existsSync( circleYmlPath ) ) {
-  const circleConfig = YAML.load( fs.readFileSync( circleYmlPath, 'utf8' ) )
-
-  const allJobs = []
-  for ( const workflow of Object.values( circleConfig.workflows ?? {} ) ) {
-    for ( const job of ( workflow.jobs ?? [] ) ) {
-      if ( typeof job === 'object' ) {
-        for ( const [ name, config ] of Object.entries( job ) ) {
-          allJobs.push( { name, config: config ?? {} } )
-        }
-      }
-    }
-  }
-
-  const coverageJob = allJobs.find( j => j.name === 'leverege/code-coverage' )
-  if ( coverageJob ) {
-    const { config } = coverageJob
-    coverageThreshold = config['coverage-threshold'] ?? DEFAULT_COVERAGE_THRESHOLD
-    cacheConfig = config['cache-config'] ?? DEFAULT_CACHE_CONFIG
-    redisEnabled = config['enable-redis'] === true
-
-    const ctx = config.context
-    const contexts = Array.isArray( ctx ) ? ctx : ( ctx ? [ ctx ] : [] )
-    enableGcp = contexts.includes( 'leverege-gcp' )
-  }
-
-  isYarnFromCircle = allJobs.some( j => j.config['pkg-manager'] === 'yarn-berry' )
-
-  log( chalk.blue( '\nParsed CircleCI config:' ) )
-  log( `  pkg-manager:        ${isYarnFromCircle ? 'yarn-berry' : 'npm'}` )
-  log( `  enable-gcp:         ${enableGcp}` )
-  log( `  coverage-threshold: ${coverageThreshold}` )
-  log( `  cache-config:       ${cacheConfig}` )
-  log( `  enable-redis:       ${redisEnabled}` )
-} else {
-  log( chalk.yellow( '\nNo .circleci/config.yml found — using defaults' ) )
-}
-
-const isYarn = isYarnFromCircle || getIsYarnProject( gitRoot )
-log( chalk.blue( `\nPackage manager: ${isYarn ? 'yarn-berry' : 'npm'}` ) )
-
-if ( redisEnabled ) {
-  log( chalk.yellow( '\nRedis detected — Redis service is always available on port 6379 in GitHub Actions' ) )
-}
 
 // ---------- Determine what needs writing ----------
 
@@ -116,13 +80,33 @@ let ciYmlExists = false
 if ( fs.existsSync( ciYmlPath ) ) {
   ciYmlExists = true
   const existingCi = fs.readFileSync( ciYmlPath, 'utf8' )
-  if ( existingCi.includes( `library-ci.yml@${GH_ACTIONS_REF}` ) ) {
+  if ( existingCi.includes( `node-ci.yml@${GH_ACTIONS_REF}` ) ) {
     log( chalk.yellow( '\nExisting GitHub Actions CI config found — will update' ) )
+    try {
+      const existingParsed = YAML.load( existingCi )
+      const existingWith = Object.values( existingParsed?.jobs ?? {} )[0]?.with ?? {}
+      if ( existingWith['enable-gcp'] != null ) enableGcp = existingWith['enable-gcp']
+      if ( existingWith['coverage-threshold'] != null ) coverageThreshold = existingWith['coverage-threshold']
+      if ( existingWith['cache-config'] != null ) cacheConfig = existingWith['cache-config']
+      log( chalk.blue( '\nPreserved from existing ci.yml:' ) )
+      log( `  enable-gcp:         ${enableGcp}` )
+      log( `  coverage-threshold: ${coverageThreshold}` )
+      log( `  cache-config:       ${cacheConfig}` )
+    } catch {
+      log( chalk.yellow( 'Could not parse existing ci.yml — using defaults' ) )
+    }
   } else {
     log( chalk.yellow( '\nExisting .github/workflows/ci.yml found (not managed by github-actions-it) — will overwrite' ) )
   }
 }
 
+if ( fs.existsSync( path.join( gitRoot, '.circleci/config.yml' ) ) ) {
+  log( chalk.yellow( '\n*** WARNING: .circleci/config.yml detected — CircleCI is deprecated. Remove it once GitHub Actions CI is confirmed green.\n' ) )
+}
+
+const isYarn = getIsYarnProject( gitRoot )
+log( chalk.blue( `\nPackage manager: ${isYarn ? 'yarn-berry' : 'npm'}` ) )
+
 // ---------- Generate content ----------
 
 function generateCiYml() {
@@ -147,11 +131,11 @@ function generateCiYml() {
     '  workflow_dispatch:',
     '',
     'jobs:',
-    '  ci:',
+    '  node-ci:',
     '    permissions:',
     '      id-token: write',
     '      contents: read',
-    `    uses: Leverege/github-actions/.github/workflows/library-ci.yml@${GH_ACTIONS_REF}`,
+    `    uses: Leverege/github-actions/.github/workflows/node-ci.yml@${GH_ACTIONS_REF}`,
   ]
 
   if ( withLines.length > 0 ) {
@@ -168,6 +152,8 @@ function generateCiYml() {
 if ( options.dryRun ) {
   log( chalk.green( `\n[dry-run] Would write ${GH_CI_YML}:\n` ) )
   log( generateCiYml() )
+  log( chalk.green( `\n[dry-run] Would write ${GH_DEPENDABOT_YML}:\n` ) )
+  log( DEPENDABOT_CONTENT )
   process.exit( 0 )
 }
 
@@ -183,5 +169,9 @@ mkdirSafe( path.join( gitRoot, GH_WORKFLOWS_DIR ) )
 fs.writeFileSync( ciYmlPath, generateCiYml() )
 log( chalk.green( `\nGitHub Actions CI config written => ${GH_CI_YML}` ) )
 
+const dependabotPath = path.join( gitRoot, GH_DEPENDABOT_YML )
+fs.writeFileSync( dependabotPath, DEPENDABOT_CONTENT )
+log( chalk.green( `Dependabot config written => ${GH_DEPENDABOT_YML}` ) )
+
 log( chalk.yellow( '\nNext steps:' ) )
-log( `  git add ${GH_CI_YML} && git commit -m "CHORE add GitHub Actions CI"\n` )
+log( `  git add ${GH_CI_YML} ${GH_DEPENDABOT_YML} && git commit -m "CHORE add GitHub Actions CI"\n` )

package/src/helm-charts/prom-operator/helmup.plugin

@@ -8,7 +8,7 @@ showInstalling "The Prometheus Operator (kube-prometheus-stack)"
 
 # https://artifacthub.io/packages/helm/prometheus-community/kube-prometheus-stack
 OCI_CHART="oci://ghcr.io/prometheus-community/charts/kube-prometheus-stack"
-[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="87.21.0"
+[ -z "$PROMETHEUS_STACK_CHART_VERSION" ] && PROMETHEUS_STACK_CHART_VERSION="88.5.4"
 helm upgrade $NS --install prometheus-stack $OCI_CHART \
                  --values  prom-operator/prometheus-stack.yaml \
                  --version $PROMETHEUS_STACK_CHART_VERSION $HELM_WHAT

package/src/helm-charts/triton-inference-leash/README.md

@@ -0,0 +1,88 @@
+# triton-inference-leash
+
+Watches for GCP zone drift between a leader pod (`triton-inference-server`) and one or more follower pods, and rolls the follower back into alignment when they diverge.
+
+## Why
+
+Inter-zone egress between co-located services costs ~$50–70/day/cluster. Pod affinity rules keep them co-located at scheduling time, but if the leader reschedules to a different zone (node failure, eviction, etc.) the existing follower pods stay put. This watcher detects the drift and rolls the follower deployment so its affinity rules re-land it in the leader's zone.
+
+## How it works
+
+Every 60 seconds the watcher:
+1. Fetches all node → zone mappings in a single `kubectl get nodes` call
+2. Looks up the zone of the running leader pod
+3. Looks up the zone of **every** running follower pod
+4. If any follower pod is outside the leader's zone → triggers `kubectl rollout restart` on the follower deployment
+5. The existing pod affinity on the follower reschedules all pods into the leader's zone
+
+A 5-minute cooldown prevents back-to-back rollouts if something is thrashing.
+
+## Deploy
+
+Before deploying, set `FOLLOWER_LABEL` and `FOLLOWER_DEPLOY` in the Deployment env block — both default to `REPLACE_ME_BEFORE_DEPLOYING` and the container will exit with an error if left unset.
+
+Deploy it:
+
+```bash
+helmup triton-inference-leash
+```
+
+Shut it down:
+
+```bash
+helmdn triton-inference-leash
+```
+
+## Debug mode
+
+Debug mode logs the actions the watcher **would** take without actually rolling the follower. Useful for validating zone detection before going live.
+
+To deploy with debug on, set `DEBUG: "true"` in the Deployment env block before applying:
+
+```yaml
+- name: DEBUG
+  value: "true"
+```
+
+Then watch the logs:
+
+```bash
+kubectl logs -f -n default -l app=triton-inference-leash
+```
+
+You should see `DRY-RUN: kubectl rollout restart ...` instead of an actual roll when a mismatch is detected. Set `DEBUG: "false"` and re-apply to go live.
+
+## Configuration
+
+| Variable | Default | Description |
+|---|---|---|
+| `NAMESPACE` | `default` | Namespace where both services run |
+| `LEADER_LABEL` | `app.kubernetes.io/instance=triton-inference-server` | Label selector for leader pods |
+| `FOLLOWER_DEPLOY` | *(required)* | Helm release name of the follower service — used as the Deployment name and to derive its pod label (`app.kubernetes.io/instance=<name>`) |
+| `POLL_INTERVAL` | `60` | Seconds between zone checks |
+| `COOLDOWN` | `300` | Seconds to wait before triggering another rollout |
+| `DEBUG` | `false` | Dry-run mode — logs actions without executing them |
+
+## Monitoring
+
+```bash
+# Tail live logs
+k8s log triton-inference-leash
+
+# Check pod status
+kubectl get pod -n default -l app=triton-inference-leash
+```
+
+Log output examples:
+```
+2026-08-20T14:00:00Z OK: leader=us-central1-b follower=[us-central1-b]
+2026-08-20T15:00:00Z OK: leader=us-central1-b follower=[us-central1-b]
+2026-08-20T15:01:00Z MISMATCH: leader=us-central1-b follower=[us-central1-b us-central1-c] — rolling my-follower-service
+2026-08-20T15:03:12Z Rollout complete
+```
+
+`OK` lines are logged once per hour in normal mode, and every poll cycle in debug mode.
+
+## Adapting for other clusters
+
+Copy the manifest, fill in `FOLLOWER_LABEL` and `FOLLOWER_DEPLOY` for the target cluster, and apply. Everything else can be left at defaults.

package/src/helm-charts/triton-inference-leash/follow-me.yaml

@@ -0,0 +1,229 @@
+# Watches for zone drift between a leader pod (triton-inference-server) and follower pods.
+# When they land in different zones, inter-zone egress costs ~$50-70/day/cluster.
+# On mismatch, rolls the follower deployment so its pod affinity re-lands it with the leader.
+#
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: triton-inference-leash
+  namespace: default
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: triton-inference-leash-nodes
+rules:
+- apiGroups: [""]
+  resources: ["nodes"]
+  verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: triton-inference-leash-nodes
+subjects:
+- kind: ServiceAccount
+  name: triton-inference-leash
+  namespace: default
+roleRef:
+  kind: ClusterRole
+  name: triton-inference-leash-nodes
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: triton-inference-leash
+  namespace: default
+rules:
+- apiGroups: [""]
+  resources: ["pods"]
+  verbs: ["get", "list"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch", "patch"]
+- apiGroups: ["apps"]
+  resources: ["replicasets"]
+  verbs: ["get", "list", "watch"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: triton-inference-leash
+  namespace: default
+subjects:
+- kind: ServiceAccount
+  name: triton-inference-leash
+  namespace: default
+roleRef:
+  kind: Role
+  name: triton-inference-leash
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: triton-inference-leash-script
+  namespace: default
+data:
+  leash.sh: |
+    #!/bin/bash
+    set -uo pipefail
+
+    NAMESPACE="${NAMESPACE:-default}"
+    LEADER_LABEL="${LEADER_LABEL:-app.kubernetes.io/instance=triton-inference-server}"
+    # FOLLOWER_LABEL: Helm release name of the follower service.
+    # Used as both the Deployment name and to derive its pod selector
+    # (app.kubernetes.io/instance=<FOLLOWER_LABEL>). Set in the Deployment env block.
+    POLL_INTERVAL="${POLL_INTERVAL:-60}"
+    COOLDOWN="${COOLDOWN:-300}"
+    DEBUG="${DEBUG:-false}"
+
+    log() { echo "$(date -u '+%Y-%m-%dT%H:%M:%SZ') $*"; }
+    dryrun() { [ "$DEBUG" = "true" ] && log "DRY-RUN: $*" || eval "$@"; }
+
+    [ -z "${FOLLOWER_LABEL:-}" ] && { log "ERROR: FOLLOWER_LABEL is required"; exit 1; }
+    FOLLOWER_SELECTOR="app.kubernetes.io/instance=${FOLLOWER_LABEL}"
+
+    # nodeName -> zone for every node, fetched in a single call per poll.
+    # kubectl process startup is a CPU burst, so the loop keeps its call count flat
+    # regardless of follower replica count.
+    declare -A NODE_ZONE
+
+    load_node_zones() {
+      local name zone
+      NODE_ZONE=()
+      while read -r name zone; do
+        [ -n "$name" ] && NODE_ZONE[$name]=$zone
+      done < <(kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.metadata.labels.topology\.kubernetes\.io/zone}{"\n"}{end}' 2>/dev/null)
+      [ "${#NODE_ZONE[@]}" -gt 0 ]
+    }
+
+    # Returns the node names of every running pod matching a label
+    get_pod_nodes() {
+      kubectl get pods -n "$NAMESPACE" -l "$1" \
+        --field-selector=status.phase=Running \
+        -o jsonpath='{.items[*].spec.nodeName}' 2>/dev/null
+    }
+
+    log "triton-inference-leash starting (poll=${POLL_INTERVAL}s cooldown=${COOLDOWN}s debug=${DEBUG})"
+    log "  leader:   ${LEADER_LABEL}"
+    log "  follower: ${FOLLOWER_SELECTOR}"
+
+    last_rollout=0
+    last_ok_log=0
+
+    while true; do
+      sleep "$POLL_INTERVAL"
+
+      load_node_zones || {
+        log "SKIP: node list unavailable"
+        continue
+      }
+
+      leader_nodes=$(get_pod_nodes "$LEADER_LABEL")
+      leader_zone=${NODE_ZONE[${leader_nodes%% *}]:-}
+      [ -z "$leader_zone" ] && {
+        log "SKIP: leader pod not running or node lookup failed"
+        continue
+      }
+
+      follower_nodes=$(get_pod_nodes "$FOLLOWER_SELECTOR")
+      [ -z "$follower_nodes" ] && {
+        log "SKIP: follower pods not running or node lookup failed"
+        continue
+      }
+
+      # Flag mismatch if any follower pod is outside the leader's zone
+      mismatch=false
+      follower_zones=""
+      for node in $follower_nodes; do
+        zone=${NODE_ZONE[$node]:-}
+        [ -z "$zone" ] && continue
+        follower_zones="${follower_zones}${zone}"$'\n'
+        [ "$zone" != "$leader_zone" ] && mismatch=true
+      done
+
+      follower_zone_summary=$(echo "$follower_zones" | sort -u | tr '\n' ' ' | xargs)
+
+      now=$(date +%s)
+
+      if [ "$mismatch" = "false" ]; then
+        if [ "$DEBUG" = "true" ] || [ $((now - last_ok_log)) -ge 3600 ]; then
+          log "OK: leader=${leader_zone} follower=[${follower_zone_summary}]"
+          last_ok_log=$now
+        fi
+        continue
+      fi
+
+      since=$((now - last_rollout))
+      if [ "$since" -lt "$COOLDOWN" ]; then
+        log "MISMATCH (leader=${leader_zone} follower=[${follower_zone_summary}]) — cooldown ${since}s/${COOLDOWN}s, skipping"
+        continue
+      fi
+
+      log "MISMATCH: leader=${leader_zone} follower=[${follower_zone_summary}] — rolling ${FOLLOWER_LABEL}"
+      dryrun kubectl rollout restart deployment/"$FOLLOWER_LABEL" -n "$NAMESPACE"
+      last_rollout=$(date +%s)
+      dryrun kubectl rollout status deployment/"$FOLLOWER_LABEL" -n "$NAMESPACE" --timeout=5m \
+        && log "Rollout complete" \
+        || log "WARNING: rollout status timed out — check deployment manually"
+    done
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: triton-inference-leash
+  namespace: default
+  labels:
+    app: triton-inference-leash
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: triton-inference-leash
+  template:
+    metadata:
+      labels:
+        app: triton-inference-leash
+    spec:
+      serviceAccountName: triton-inference-leash
+      terminationGracePeriodSeconds: 10
+      containers:
+      - name: leash
+        image: us-docker.pkg.dev/leverege-registry/system/images/leverege-kubectl:v1.31.0
+        command: ["/bin/bash", "/scripts/leash.sh"]
+        env:
+        - name: NAMESPACE
+          value: "default"
+        - name: LEADER_LABEL
+          value: "app.kubernetes.io/instance=triton-inference-server"
+        # Helm release name of the follower service — e.g. pitcrew-vision-logic-server
+        - name: FOLLOWER_LABEL
+          value: "REPLACE_ME_BEFORE_DEPLOYING"
+        - name: POLL_INTERVAL
+          value: "60"
+        - name: COOLDOWN
+          value: "300"
+        - name: DEBUG
+          value: "false"
+        resources:
+          # No cpu limit on purpose. Each kubectl invocation is a sub-second
+          # full-core burst, so any CFS ceiling below a core clips nearly every
+          # period the container is awake and keeps ContainerHighThrottleRate
+          # firing on an otherwise idle pod. The request carries the scheduling
+          # weight; memory stays capped.
+          requests:
+            cpu: 50m
+            memory: 64Mi
+          limits:
+            memory: 128Mi
+        volumeMounts:
+        - name: scripts
+          mountPath: /scripts
+      volumes:
+      - name: scripts
+        configMap:
+          name: triton-inference-leash-script
+          defaultMode: 0755

package/src/helm-charts/triton-inference-leash/helmdn.plugin

@@ -0,0 +1,3 @@
+#!/bin/bash
+#
+kubectl delete -f triton-inference-leash/follow-me.yaml

package/src/helm-charts/triton-inference-leash/helmup.plugin

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+showInstalling "Triton Inference Leash"
+
+if grep -q "REPLACE_ME_BEFORE_DEPLOYING" triton-inference-leash/follow-me.yaml; then
+  echo "ERROR: follow-me.yaml still contains placeholder values — set FOLLOWER_LABEL and FOLLOWER_DEPLOY before deploying"
+  exit 1
+fi
+
+kubectl apply -f triton-inference-leash/follow-me.yaml $K8S_WHAT
+
+cat<<MONITOR_LEASH
+
+Monitor with => `color g "kubectl logs -f -n default -l app=triton-inference-leash"`
+
+MONITOR_LEASH

package/src/k8scale.sh

@@ -45,8 +45,10 @@ do
   RESULTS="`color g SUCCESS`"
   if [ "$DIRECTION" == "roll" ];
   then
+    btDeprecated "See => `color g 'k8s roll --help'`"
     kubectl rollout restart deployment $SERVICE &> $DEVNULL
   else
+    btDeprecated "See => `color g 'k8s scale --help'`"
     kubectl scale --replicas=$REPLICAS deployment $SERVICE &> $DEVNULL
   fi
   [ $? -eq 1 ] && RESULTS="`color r FAILURE`"

package/src/k8x.sh

@@ -2,6 +2,8 @@
 
 . `build-tools --bashfun`
 
+btDeprecated "See => `color g 'k8s exe --help'`"
+
 if [ ! -x "$(command -v fzf)" ];
 then
   cat<<NO_FZF

package/src/klog.sh

@@ -1,11 +1,15 @@
 #!/bin/bash
 
+. `build-tools --bashfun`
+
 overwhelm
 
 DTS="`date +\"%Y%m%d-%H%M%S\"`"
 PRETTY="pino-pretty -c -t 'HH:MM:ss.l' -i 'severity'"
 IFS="/" read APP <<< "$1"
 
+btDeprecated "See => `color g 'k8s log --help'`"
+
 if [ ! -z "$APP" ];
 then
   NS=${2:-"default"}

package/src/push-my-chart.mjs

@@ -11,6 +11,7 @@ import {
   parseHelmChart,
   proceed,
   shellCmd,
+  sleep,
   warning,
 } from './Utils.mjs'
 
@@ -29,6 +30,19 @@ without disrupting current configurations.` ) )
 program.parse( process.argv )
 const opts = program.opts()
 
+// DEPRECATED
+log( `
+  ${chalk.red.bold( '*** DEPRECATED *** the museum is closing *** DEPRECATED ***' )}
+
+  ${chalk.yellow( 'modify the package.json museum script to use chart-to-registry instead' )}
+
+  ${chalk.green.bold( '  "museum": "chart-to-registry",' )}
+
+` )
+
+await sleep( 3000 )
+// DEPRECATED
+
 const {
   chartName,
   chartVersion,

package/src/templates/dockerfileTemplateNodeJs.hbs

@@ -70,6 +70,7 @@ RUN npm ci --only=production --no-optional {{npmlogging}} && \
 # --------------------------------------------------------------
 
 FROM {{image}}
+{{finalStagePreInstallPluginfile}}
 
 ARG BUILD_ENV="{{buildEnv}}"
 
@@ -97,7 +98,7 @@ RUN if [ "$BUILD_ENV" = "debian" ]; then \
       npm install -g {{npmVersion}} && \
       {{/if}}
       rm /bin/sh && ln -s /bin/bash /bin/sh && \
-      mkdir -p /usr/src/app /tmp/levlog && chown node:node /usr/src/app; \
+      mkdir -p /usr/src/app /tmp/levlog && chown {{runuser}}:{{runuser}} /usr/src/app; \
     else \
       apk update && \
       apk add --no-cache bash curl tini vim {{apkadds}} && \
@@ -108,7 +109,7 @@ RUN if [ "$BUILD_ENV" = "debian" ]; then \
       npm install -g {{npmVersion}} && \
       {{/if}}
       rm /bin/sh && ln -s /bin/bash /bin/sh && \
-      mkdir -p /usr/src/app /tmp/levlog && chown node:node /usr/src/app; \
+      mkdir -p /usr/src/app /tmp/levlog && chown {{runuser}}:{{runuser}} /usr/src/app; \
     fi
 
 
@@ -125,8 +126,8 @@ RUN rm -f ${NPM_CONFIG_USERCONFIG}
 ENTRYPOINT [ "tini", "--" ]
 WORKDIR /usr/src/app
 
-COPY --chown=node:node --from=intermediate /usr/src/app /usr/src/app
+COPY --chown={{runuser}}:{{runuser}} --from=intermediate /usr/src/app /usr/src/app
 
 USER {{runuser}}
-COPY ./bashrc /home/node/.bashrc
-CMD [ "/bin/bash", "-c", "source /home/node/.bashrc && {{runCommand}}" ]
+COPY --chown={{runuser}}:{{runuser}} ./bashrc /home/{{runuser}}/.bashrc
+CMD [ "/bin/bash", "-c", "source /home/{{runuser}}/.bashrc && {{runCommand}}" ]