@underpostnet/cyberia 3.3.73 → 3.3.77

package/src/cli/deploy.js

@@ -50,6 +50,8 @@ import nodePath from 'node:path';
 import dotenv from 'dotenv';
 import os from 'node:os';
 import crypto from 'node:crypto';
+import { fileURLToPath } from 'node:url';
+import { dump as yamlDump, load as yamlLoad } from 'js-yaml';
 import { appSecretName } from './app.js';
 import { domainContextFactory } from './domains.js';
 import Underpost from '../index.js';
@@ -93,6 +95,23 @@ const UPSTREAM_FAILURE_STATUSES = [502, 503, 504];
 
 const CONTAINER_ENGINE_ROOT = '/home/dd/engine';
 
+/**
+ * @constant LETSENCRYPT_ISSUER_NAME
+ * @description The ClusterIssuer every generated Certificate references.
+ * @memberof UnderpostDeploy
+ */
+const LETSENCRYPT_ISSUER_NAME = 'letsencrypt-prod';
+
+/**
+ * @constant LETSENCRYPT_ISSUER_MANIFEST
+ * @description Base ClusterIssuer manifest: account, ACME server and key Secret. Resolved
+ * from this module so an installed CLI and a checkout read the same file.
+ * @memberof UnderpostDeploy
+ */
+const LETSENCRYPT_ISSUER_MANIFEST = fileURLToPath(
+  new URL(`../../manifests/${LETSENCRYPT_ISSUER_NAME}.yaml`, import.meta.url),
+);
+
 // The manifest set `deploy --build-manifest` produces for one environment.
 const DEFAULT_MANIFEST_FILES = [
   'deployment.yaml',
@@ -147,10 +166,10 @@ const GATEWAY_DURATION_UNITS = [
 ];
 
 /**
- * Converts an HTTPProxy duration (`300000ms`, `10s`, `infinity`) into a Gateway
+ * Converts an HTTPProxy duration (`10000ms`, `10s`, `infinity`) into a Gateway
  * API Duration. The Gateway API grammar allows at most 5 digits per component,
  * so a value that overflows in one unit is re-expressed in a coarser one
- * (`300000ms` → `5m`); `infinity` maps to `0s`, which disables the timeout.
+ * (`10000ms` → `5m`); `infinity` maps to `0s`, which disables the timeout.
  * @param {string|number} value - Source duration.
  * @returns {string|null} Gateway API Duration, or null when unset/unparsable.
  */
@@ -255,12 +274,68 @@ ${buildKindPorts(fromPort, toPort)}`;
           : Underpost.deploy.trafficServiceYamlFactory({ deployId, env, traffic, namespace, fromPort, toPort });
       const escaped = `${deployId}-${env}`.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
       manifest = manifest.replace(new RegExp(`app: ${escaped}-(?:blue|green)`), `app: ${deployId}-${env}-${traffic}`);
-      shellExec(`kubectl apply -f - -n ${namespace} <<'EOF'
+      logger.info('Applying the traffic Service', { deployId, env, traffic, namespace });
+      shellExec(
+        `kubectl apply -f - -n ${namespace} <<'EOF'
 ${manifest}
 EOF
-`);
+`,
+        { disableLog: true },
+      );
       return Underpost.deploy.trafficServiceNameFactory({ deployId, env });
     },
+    /**
+     * Removes the Gateway API objects an earlier model of this deploy left behind, so
+     * they stop competing with the consolidated `<deployId>-<env>` Gateway.
+     *
+     * The per-host model emitted one Gateway and one `<host>-http3` ClientTrafficPolicy
+     * per hostname. Left in place they duplicate that hostname's listeners in the same
+     * merged set, and the oldest resource retains traffic. `undefined` / `undefined-http3`
+     * is the same problem under a different name: the consolidated objects were briefly
+     * emitted with an unresolved host in their metadata, and merged listeners are
+     * configured by the oldest policy that targets them.
+     *
+     * The namespace is listed once and only objects that exist are deleted: a sweep that
+     * issued a blind delete per host per kind spent two dozen `kubectl` round-trips per
+     * sync on names that were gone after the first migration, and buried the one line
+     * that mattered when something was actually removed.
+     * @param {object} params
+     * @param {string[]} params.hosts - Hostnames the deploy's `conf.server.json` declares.
+     * @param {string} params.namespace - Namespace the deploy's Gateway objects live in.
+     * @returns {{gateways: string[], policies: string[]}} Names removed, per kind.
+     * @memberof UnderpostDeploy
+     */
+    sweepLegacyGatewayObjects({ hosts = [], namespace = 'default' }) {
+      const existing = (kind) =>
+        new Set(
+          `${
+            shellExec(`kubectl get ${kind} -n ${namespace} -o name --ignore-not-found`, {
+              stdout: true,
+              silent: true,
+              silentOnError: true,
+              disableLog: true,
+            }) ?? ''
+          }`
+            .split('\n')
+            .map((line) => line.trim().split('/').pop())
+            .filter(Boolean),
+        );
+      const legacy = [...new Set(hosts.filter(Boolean)), 'undefined'];
+      const gateways = existing('Gateway');
+      const policies = existing('ClientTrafficPolicy');
+      const removed = {
+        gateways: legacy.filter((name) => gateways.has(name)),
+        policies: legacy.map((name) => `${name}-http3`).filter((name) => policies.has(name)),
+      };
+      for (const name of removed.gateways)
+        shellExec(`sudo kubectl delete Gateway ${name} -n ${namespace} --ignore-not-found`);
+      for (const name of removed.policies)
+        shellExec(`sudo kubectl delete ClientTrafficPolicy ${name} -n ${namespace} --ignore-not-found`);
+      if (removed.gateways.length > 0 || removed.policies.length > 0)
+        logger.warn('Legacy per-host Gateway API objects removed', { namespace, ...removed });
+      return removed;
+    },
+
     /**
      * Removes the route kind owned by the inactive ingress stack. The active
      * route is already published before this runs, so migration never creates a
@@ -617,7 +692,7 @@ spec:
      * @param {string} options.namespace - Kubernetes namespace for the deployment (defaults to "default").
      * @param {string} [options.versions] - Comma-separated list of versions to deploy.
      * @param {string} [options.cmd] - Custom initialization command for deploymentYamlPartsFactory (comma-separated commands).
-     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "300000ms", "infinity").
+     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "10000ms", "infinity").
      * @param {string} [options.timeoutIdle] - HTTPProxy per-route idle timeout (e.g. "10s", "infinity").
      * @param {string} [options.retryCount] - HTTPProxy per-route retry count (e.g. 3).
      * @param {string} [options.retryPerTryTimeout] - HTTPProxy per-route per-try timeout (e.g. "150ms").
@@ -1112,6 +1187,125 @@ spec:
     kind: ClusterIssuer
   secretName: ${host}`;
     },
+
+    /**
+     * The Gateways an ACME HTTP-01 challenge route must attach to: every Gateway the
+     * namespace holds now, plus the one each routed deploy publishes under, whether or
+     * not it exists yet. A challenge HTTPRoute lists them all as parents; the data plane
+     * attaches it to whichever one carries a listener for the challenged hostname, and
+     * reports the rest as unmatched, which is harmless.
+     *
+     * The union rather than either side alone: the live list covers Gateways a deploy
+     * outside `dd.routes` created, the route table covers a deploy that has not been
+     * applied yet — an issuer that only knew the live list would have to be re-applied
+     * after every first deploy before that deploy's certificate could issue.
+     * @param {object} params
+     * @param {string} [params.namespace] - Namespace the Gateways live in.
+     * @param {string} [params.env] - Environment the routed deploys publish under.
+     * @returns {Array<{name: string, namespace: string}>} Parent references, sorted by name.
+     * @memberof UnderpostDeploy
+     */
+    acmeGatewayParentRefsFactory({ namespace = 'default', env = 'production' } = {}) {
+      const live = `${
+        shellExec(`kubectl get gateway -n ${namespace} -o name --ignore-not-found`, {
+          stdout: true,
+          silent: true,
+          silentOnError: true,
+          disableLog: true,
+        }) ?? ''
+      }`
+        .split('\n')
+        .map((line) => line.trim().split('/').pop())
+        .filter(Boolean);
+      const routed = readDeployRoutes().map((deployId) => Underpost.deploy.gatewayNameFactory({ deployId, env }));
+      return [...new Set([...live, ...routed])].sort().map((name) => ({ name, namespace }));
+    },
+
+    /**
+     * Renders the Let's Encrypt ClusterIssuer for the ingress stack that actually
+     * carries the HTTP-01 challenge.
+     *
+     * The base manifest fixes the account (email, ACME server, key Secret); only the
+     * solver is decided here. With the Gateway API stack, the challenge must be answered
+     * through an HTTPRoute on the deploy's Gateway: an `ingress`-class solver creates an
+     * Ingress nothing on the request path reads, so `/.well-known/acme-challenge/*` is
+     * routed by the application's own HTTPRoute to the workload — a 404 — or, with no
+     * workload behind it, to the maintenance fallback. Every renewal fails that way while
+     * the solver pods sit there ready, and the failure only surfaces when the certificate
+     * expires.
+     * @param {object} params
+     * @param {'gateway'|'ingress'} params.solver - Which stack answers the challenge.
+     * @param {Array<{name: string, namespace: string}>} [params.parentRefs] - Gateways for the `gateway` solver.
+     * @param {string} [params.basePath] - Base ClusterIssuer manifest.
+     * @returns {string} ClusterIssuer YAML.
+     * @throws {Error} When the Gateway solver is requested with no Gateway to attach to.
+     * @memberof UnderpostDeploy
+     */
+    letsEncryptIssuerYamlFactory({ solver, parentRefs = [], basePath = LETSENCRYPT_ISSUER_MANIFEST }) {
+      const issuer = yamlLoad(fs.readFileSync(basePath, 'utf8'));
+      if (solver === 'gateway') {
+        if (parentRefs.length === 0)
+          throw new Error('[cert-manager] the Gateway API HTTP-01 solver needs at least one Gateway parentRef');
+        issuer.spec.acme.solvers = [
+          {
+            http01: {
+              gatewayHTTPRoute: {
+                parentRefs: parentRefs.map(({ name, namespace }) => ({ name, namespace, kind: 'Gateway' })),
+              },
+            },
+          },
+        ];
+      } else issuer.spec.acme.solvers = [{ http01: { ingress: { class: 'contour' } } }];
+      return yamlDump(issuer, { lineWidth: -1, noRefs: true });
+    },
+
+    /**
+     * Whether the Let's Encrypt ClusterIssuer is installed. False both when it was never
+     * applied and when cert-manager's CRDs are absent, which is the same answer here.
+     * @returns {boolean}
+     * @memberof UnderpostDeploy
+     */
+    letsEncryptIssuerExists() {
+      return (
+        `${
+          shellExec(`kubectl get clusterissuer ${LETSENCRYPT_ISSUER_NAME} -o name --ignore-not-found`, {
+            stdout: true,
+            silent: true,
+            silentOnError: true,
+            disableLog: true,
+          }) ?? ''
+        }`.trim().length > 0
+      );
+    },
+
+    /**
+     * Applies the Let's Encrypt ClusterIssuer for the stack in use, so certificates keep
+     * renewing through whichever data plane the deploy's hostnames are served by.
+     * @param {object} params
+     * @param {'gateway'|'ingress'} params.solver - Which stack answers the challenge.
+     * @param {string} [params.namespace] - Namespace the Gateways live in.
+     * @param {string} [params.env] - Environment the routed deploys publish under.
+     * @param {string} [params.basePath] - Base ClusterIssuer manifest.
+     * @returns {{solver: string, parentRefs: Array<{name: string, namespace: string}>}} What was applied.
+     * @memberof UnderpostDeploy
+     */
+    applyLetsEncryptIssuer({ solver, namespace = 'default', env = 'production', basePath } = {}) {
+      const parentRefs = solver === 'gateway' ? Underpost.deploy.acmeGatewayParentRefsFactory({ namespace, env }) : [];
+      const manifest = Underpost.deploy.letsEncryptIssuerYamlFactory({ solver, parentRefs, basePath });
+      const stagePath = `/dev/shm/underpost-${LETSENCRYPT_ISSUER_NAME}.yaml`;
+      fs.writeFileSync(stagePath, manifest, 'utf8');
+      try {
+        shellExec(`sudo kubectl apply -f ${stagePath}`);
+      } finally {
+        fs.removeSync(stagePath);
+      }
+      logger.info("Let's Encrypt ClusterIssuer applied", {
+        issuer: LETSENCRYPT_ISSUER_NAME,
+        solver,
+        gateways: parentRefs.map(({ name }) => name),
+      });
+      return { solver, parentRefs };
+    },
     /**
      * Mirrors each deploy's built development manifests into the project tree, and refreshes the
      * public default configuration they belong to.
@@ -2185,7 +2379,7 @@ ${rules}`;
      * @param {boolean} options.disableUpdateVolume - Whether to disable volume updates.
      * @param {boolean} options.disableUpdateUnderpostConfig - Whether to disable Underpost config updates.
      * @param {string} [options.namespace] - Kubernetes namespace for the deployment (defaults to "default").
-     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "300000ms", "infinity").
+     * @param {string} [options.timeoutResponse] - HTTPProxy per-route response timeout (e.g. "10000ms", "infinity").
      * @param {string} [options.timeoutIdle] - HTTPProxy per-route idle timeout (e.g. "10s", "infinity").
      * @param {string} [options.retryCount] - HTTPProxy per-route retry count (e.g. 3).
      * @param {string} [options.retryPerTryTimeout] - HTTPProxy per-route per-try timeout (e.g. "150ms").
@@ -2341,34 +2535,15 @@ EOF`);
                 });
           }
 
-        for (const host of Object.keys(confServer)) {
-          if (!options.disableUpdateProxy) {
-            // The host's route object is left in place and replaced by the `apply`
-            // below. Deleting it first unpublished the hostname for the whole
-            // reconciliation window, so every promote dropped live requests before
-            // the new colour was ever the question.
-            //
-            // A deploy that previously ran the per-host model left a Gateway
-            // named after each host. Those are superseded by the consolidated
-            // one, and leaving them behind duplicates that hostname's listeners
-            // in the same merged set. The oldest resource would retain traffic.
-            //
-            // `undefined-http3` is the same problem under a different name: the
-            // consolidated policy was briefly emitted with an unresolved host in
-            // its metadata. Merged listeners are configured by the oldest policy
-            // that targets them, so that object outranks the correctly named one
-            // for as long as it exists.
-            if (options.gatewayApi)
-              for (const name of [host, 'undefined']) {
-                shellExec(`sudo kubectl delete Gateway ${name} -n ${namespace} --ignore-not-found`, { silent: true });
-                shellExec(`sudo kubectl delete ClientTrafficPolicy ${name}-http3 -n ${namespace} --ignore-not-found`, {
-                  silent: true,
-                });
-              }
-            if (Underpost.deploy.isCertManagerContext({ host, env, options }))
-              shellExec(`sudo kubectl delete Certificate ${host} -n ${namespace} --ignore-not-found`);
-          }
-        }
+        // The host's route object is left in place and replaced by the `apply`
+        // below. Deleting it first unpublished the hostname for the whole
+        // reconciliation window, so every promote dropped live requests before
+        // the new colour was ever the question.
+        if (!options.disableUpdateProxy && options.gatewayApi)
+          Underpost.deploy.sweepLegacyGatewayObjects({ hosts: Object.keys(confServer), namespace });
+        for (const host of Object.keys(confServer))
+          if (!options.disableUpdateProxy && Underpost.deploy.isCertManagerContext({ host, env, options }))
+            shellExec(`sudo kubectl delete Certificate ${host} -n ${namespace} --ignore-not-found`);
 
         const manifestsPath =
           env === 'production'
@@ -2480,6 +2655,11 @@ EOF`);
           }
 
           if (Underpost.deploy.isCertManagerContext({ host: Object.keys(confServer)[0], env, options })) {
+            // The issuer's challenge route must list this deploy's Gateway as a parent, or
+            // the certificates applied below can never issue. Re-rendered from the live
+            // Gateway list, so a deploy created after `cluster --cert-manager` is covered.
+            if (options.gatewayApi && Underpost.deploy.letsEncryptIssuerExists())
+              Underpost.deploy.applyLetsEncryptIssuer({ solver: 'gateway', namespace, env });
             const secretPath = `./${manifestsPath}/secret.yaml`;
             if (fs.existsSync(secretPath) && fs.readFileSync(secretPath, 'utf8').trim()) {
               shellExec(`sudo kubectl apply -f ${secretPath} -n ${namespace}`);
@@ -3143,3 +3323,4 @@ spec:
 }
 
 export default UnderpostDeploy;
+export { LETSENCRYPT_ISSUER_MANIFEST, LETSENCRYPT_ISSUER_NAME };

package/src/cli/docker-compose.js

@@ -192,7 +192,7 @@ services:
 MONGO_IMAGE=mongo:latest
 VALKEY_IMAGE=valkey/valkey:latest
 APP_IMAGE=underpost/underpost-engine
-APP_TAG=v3.3.73
+APP_TAG=v3.3.77
 PROXY_IMAGE=nginx:stable-alpine
 PROMETHEUS_IMAGE=prom/prometheus:latest
 GRAFANA_IMAGE=grafana/grafana:latest

package/src/cli/fs-selection.js

@@ -0,0 +1,115 @@
+import fs from 'fs-extra';
+import path from 'node:path';
+import UnderpostRepository from './repository.js';
+
+const manifestFilterNames = ['key', 'fromKey', 'toKey', 'keyRegex'];
+
+const normalizeStoragePath = (value) => {
+  if (value === undefined || value === '') return '';
+  const absolute = path.resolve(value);
+  const relative = path.relative(process.cwd(), absolute);
+  return (
+    path.isAbsolute(value) && (relative === '..' || relative.startsWith(`..${path.sep}`)) ? absolute : relative || '.'
+  )
+    .split(path.sep)
+    .join('/');
+};
+
+const isWithinScope = (key, scope) => {
+  if (scope === undefined || scope === '') return true;
+  const relative = path.relative(path.resolve(scope), path.resolve(key));
+  return relative !== '..' && !relative.startsWith(`..${path.sep}`) && !path.isAbsolute(relative);
+};
+
+const validateSelectionOptions = (scope, options) => {
+  if (options.git && options.tracked) throw new Error('Use either --git or --tracked.');
+  if (!options.tracked && !scope) throw new Error('Provide a filesystem path, or use --tracked.');
+  if (options.rm && options.pull) throw new Error('Use either --rm or --pull.');
+  if (options.omitUnzip && !options.pull) throw new Error('--omit-unzip requires --pull.');
+};
+
+const filterManifestEntries = (entries, options = {}, scope) => {
+  const keys = entries.map(([key]) => key);
+  const keyIndex = (name) => {
+    const index = keys.indexOf(options[name]);
+    if (index < 0) throw new Error(`Manifest key not found for ${name}: ${options[name]}`);
+    return index;
+  };
+  const start = options.fromKey === undefined ? 0 : keyIndex('fromKey');
+  const end = options.toKey === undefined ? keys.length - 1 : keyIndex('toKey');
+  if (start > end && (options.fromKey !== undefined || options.toKey !== undefined))
+    throw new Error('--from-key must precede or equal --to-key.');
+
+  let regex;
+  if (options.keyRegex !== undefined) {
+    try {
+      regex = new RegExp(options.keyRegex);
+    } catch {
+      throw new Error(`Invalid --key-regex: ${options.keyRegex}`);
+    }
+  }
+  const matches = (key) => isWithinScope(key, scope) && (!regex || regex.test(key));
+  if (options.key !== undefined) {
+    const index = keyIndex('key');
+    if (index < start || index > end || !matches(options.key))
+      throw new Error('--key conflicts with the manifest range, path scope, or regex.');
+  }
+  return entries
+    .slice(start, end + 1)
+    .filter(([key]) => matches(key) && (options.key === undefined || key === options.key));
+};
+
+const resolveFilesystemSelection = (scope) => {
+  const files = [];
+  const visit = (target) => {
+    const stats = fs.statSync(target);
+    if (stats.isFile()) files.push(normalizeStoragePath(target));
+    else if (stats.isDirectory()) {
+      for (const entry of fs
+        .readdirSync(target, { withFileTypes: true })
+        .sort((a, b) => a.name.localeCompare(b.name))) {
+        const child = path.join(target, entry.name);
+        // Do not follow directory links during traversal.
+        if (entry.isSymbolicLink()) {
+          if (fs.statSync(child, { throwIfNoEntry: false })?.isFile()) files.push(normalizeStoragePath(child));
+        } else visit(child);
+      }
+    } else throw new Error(`Storage requires a regular file or directory: ${target}`);
+  };
+  visit(scope);
+  return files;
+};
+
+const resolveGitSelection = (files, scope) => {
+  const context = fs.statSync(scope).isDirectory() ? scope : path.dirname(scope);
+  const tracked = new Set(UnderpostRepository.API.getTrackedFiles(context));
+  return files.filter((file) => tracked.has(path.resolve(file)));
+};
+
+const resolveTrackedSelection = (entries) => entries.map(([key]) => key);
+
+const resolveSelection = (scope, manifest, options = {}) => {
+  validateSelectionOptions(scope, options);
+  const entries = Object.entries(manifest);
+  const filtered = filterManifestEntries(entries, options, scope);
+  if (options.tracked) return resolveTrackedSelection(filtered);
+
+  let files = resolveFilesystemSelection(scope);
+  if (options.git) files = resolveGitSelection(files, scope);
+  const manifestKeys = new Map(entries.map(([key]) => [normalizeStoragePath(key), key]));
+  files = files.map((file) => manifestKeys.get(file) ?? file);
+  if (manifestFilterNames.some((name) => options[name] !== undefined)) {
+    const selected = new Set(resolveTrackedSelection(filtered));
+    files = files.filter((file) => selected.has(file));
+  }
+  return files;
+};
+
+export {
+  normalizeStoragePath,
+  filterManifestEntries,
+  resolveFilesystemSelection,
+  resolveGitSelection,
+  resolveTrackedSelection,
+  resolveSelection,
+};