@underpostnet/underpost 2.99.8 → 3.0.0

package/src/cli/index.js

@@ -214,6 +214,7 @@ program
   .option('--postgresql', 'Initializes the cluster with a PostgreSQL statefulset.')
   .option('--mongodb4', 'Initializes the cluster with a MongoDB 4.4 service.')
   .option('--valkey', 'Initializes the cluster with a Valkey service.')
+  .option('--ipfs', 'Initializes the cluster with an ipfs-cluster statefulset.')
   .option('--contour', 'Initializes the cluster with Project Contour base HTTPProxy and Envoy.')
   .option('--cert-manager', "Initializes the cluster with a Let's Encrypt production ClusterIssuer.")
   .option('--dedicated-gpu', 'Initializes the cluster with dedicated GPU base resources and environment settings.')
@@ -242,12 +243,12 @@ program
   .option('--init-host', 'Installs necessary Kubernetes node CLI tools (e.g., kind, kubeadm, docker, podman, helm).')
   .option('--uninstall-host', 'Uninstalls all host components installed by init-host.')
   .option('--config', 'Sets the base Kubernetes node configuration.')
-  .option('--worker', 'Sets the context for a worker node.')
   .option('--chown', 'Sets the appropriate ownership for Kubernetes kubeconfig files.')
   .option('--k3s', 'Initializes the cluster using K3s (Lightweight Kubernetes).')
   .option('--hosts <hosts>', 'A comma-separated list of cluster hostnames or IP addresses.')
   .option('--remove-volume-host-paths', 'Removes specified volume host paths after execution.')
   .option('--namespace <namespace>', 'Kubernetes namespace for cluster operations (defaults to "default").')
+  .option('--replicas <replicas>', 'Sets a custom number of replicas for statefulset deployments.')
   .action(Underpost.cluster.init)
   .description('Manages Kubernetes clusters, defaulting to Kind cluster initialization.');
 
@@ -295,6 +296,10 @@ program
   .option('--namespace <namespace>', 'Kubernetes namespace for deployment operations (defaults to "default").')
   .option('--kind-type <kind-type>', 'Specifies the Kind cluster type for deployment operations.')
   .option('--port <port>', 'Sets up port forwarding from local to remote ports.')
+  .option(
+    '--expose-port <port>',
+    'Sets the local:remote port to expose when --expose is active (overrides auto-detected service port).',
+  )
   .option('--cmd <cmd>', 'Custom initialization command for deployment (comma-separated commands).')
   .description('Manages application deployments, defaulting to deploying development pods.')
   .action(Underpost.deploy.callback);
@@ -613,37 +618,33 @@ program
 
 program
   .command('lxd')
-  .option('--init', 'Initializes LXD on the current machine.')
-  .option('--reset', 'Resets LXD on the current machine, deleting all configurations.')
-  .option('--install', 'Installs LXD on the current machine.')
-  .option('--dev', 'Sets the development context environment for LXD.')
-  .option('--create-virtual-network', 'Creates an LXD virtual network bridge.')
-  .option('--create-admin-profile', 'Creates an admin profile for LXD management.')
-  .option('--control', 'Sets the context for a control node VM.')
-  .option('--worker', 'Sets the context for a worker node VM.')
-  .option('--create-vm <vm-id>', 'Creates default virtual machines with the specified ID.')
-  .option('--init-vm <vm-id>', 'Retrieves the Underpost initialization script for the specified VM.')
-  .option('--info-vm <vm-id>', 'Retrieves all information about the specified VM.')
-  .option('--test <vm-id>', 'Tests the health, status, and network connectivity for a VM.')
-  .option('--root-size <gb-size>', 'Sets the root partition size (in GB) for the VM.')
-  .option('--k3s', 'Flag to indicate that the VM initialization is for a K3s cluster type.')
+  .option('--init', 'Initializes LXD on the current machine via preseed.')
+  .option('--reset', 'Removes the LXD snap and purges all data.')
+  .option('--install', 'Installs the LXD snap.')
+  .option('--dev', 'Use local paths instead of the global npm installation.')
+  .option('--create-virtual-network', 'Creates the lxdbr0 bridge network.')
+  .option('--ipv4-address <cidr>', 'IPv4 address/CIDR for the lxdbr0 bridge network (default: "10.250.250.1/24").')
+  .option('--create-admin-profile', 'Creates the admin-profile for VM management.')
+  .option('--control', 'Initialize the target VM as a K3s control plane node.')
+  .option('--worker', 'Initialize the target VM as a K3s worker node.')
+  .option('--create-vm <vm-name>', 'Copy the LXC launch command for a new K3s VM to the clipboard.')
+  .option('--delete-vm <vm-name>', 'Stop and delete the specified VM.')
+  .option('--init-vm <vm-name>', 'Run k3s-node-setup.sh on the specified VM (use with --control or --worker).')
+  .option('--info-vm <vm-name>', 'Display full configuration and status for the specified VM.')
+  .option('--test <vm-name>', 'Run connectivity and health checks on the specified VM.')
+  .option('--root-size <gb-size>', 'Root disk size in GiB for --create-vm (default: 32).')
   .option(
     '--join-node <nodes>',
-    'A comma-separated list of worker and control nodes to join (e.g., "k8s-worker-1,k8s-control").',
-  )
-  .option(
-    '--expose <vm-name-ports>',
-    'Exposes specified ports on a VM (e.g., "k8s-control:80,443"). Multiple VM-port pairs can be comma-separated.',
-  )
-  .option(
-    '--delete-expose <vm-name-ports>',
-    'Removes exposed ports on a VM (e.g., "k8s-control:80,443"). Multiple VM-port pairs can be comma-separated.',
-  )
-  .option('--workflow-id <workflow-id>', 'Sets the workflow ID context for LXD operations.')
-  .option('--vm-id <vm-id>', 'Sets the VM ID context for LXD operations.')
-  .option('--deploy-id <deploy-id>', 'Sets the deployment ID context for LXD operations.')
-  .option('--namespace <namespace>', 'Kubernetes namespace for LXD operations (defaults to "default").')
-  .description('Manages LXD containers and virtual machines.')
+    'Join a K3s worker to a control plane. Standalone format: "workerName,controlName". ' +
+      'When used with --init-vm --worker, provide just the control node name for auto-join.',
+  )
+  .option('--expose <vm-name:ports>', 'Proxy host ports to a VM (e.g., "k3s-control:80,443").')
+  .option('--delete-expose <vm-name:ports>', 'Remove proxied ports from a VM (e.g., "k3s-control:80,443").')
+  .option('--workflow-id <workflow-id>', 'Workflow ID to execute via runWorkflow.')
+  .option('--vm-id <vm-name>', 'Target VM name for workflow execution.')
+  .option('--deploy-id <deploy-id>', 'Deployment ID context for workflow execution.')
+  .option('--namespace <namespace>', 'Kubernetes namespace context (defaults to "default").')
+  .description('Manages LXD virtual machines as K3s nodes (control plane or workers).')
   .action(Underpost.lxd.callback);
 
 program

package/src/cli/ipfs.js

@@ -0,0 +1,184 @@
+/**
+ * IPFS Cluster module for managing ipfs-cluster StatefulSet deployment on Kubernetes.
+ * @module src/cli/ipfs.js
+ * @namespace UnderpostIPFS
+ */
+
+import { loggerFactory } from '../server/logger.js';
+import { shellExec } from '../server/process.js';
+import fs from 'fs-extra';
+import Underpost from '../index.js';
+
+const logger = loggerFactory(import.meta);
+
+/**
+ * @class UnderpostIPFS
+ * @description Manages deployment of an ipfs-cluster StatefulSet on Kubernetes.
+ * Credentials (cluster secret + peer identity) are generated once and persisted
+ * to engine-private/ so the cluster identity survives redeployments.
+ * @memberof UnderpostIPFS
+ */
+class UnderpostIPFS {
+  static API = {
+    /**
+     * @method resolveCredentials
+     * @description Resolves the IPFS cluster credentials from engine-private/ if they
+     * already exist, otherwise generates new ones (hex cluster secret + peer identity
+     * via ipfs-cluster-service init) and persists them with mode 0o600.
+     * @param {string} privateDir - Absolute path to the engine-private directory.
+     * @returns {{ CLUSTER_SECRET: string, IDENTITY_JSON: { id: string, private_key: string } }}
+     * @memberof UnderpostIPFS
+     */
+    resolveCredentials(privateDir) {
+      const secretPath = `${privateDir}/ipfs-cluster-secret`;
+      const identityPath = `${privateDir}/ipfs-cluster-identity.json`;
+
+      if (fs.existsSync(secretPath) && fs.existsSync(identityPath)) {
+        logger.info('Reusing existing IPFS cluster credentials from engine-private/');
+        return {
+          CLUSTER_SECRET: fs.readFileSync(secretPath, 'utf8').trim(),
+          IDENTITY_JSON: JSON.parse(fs.readFileSync(identityPath, 'utf8')),
+        };
+      }
+
+      logger.info('Generating new IPFS cluster credentials and persisting to engine-private/');
+
+      // ipfs-cluster-service requires CLUSTER_SECRET as a 64-char hex string.
+      // base64 (openssl rand -base64 32) contains '/', '+', '=' which are invalid hex bytes.
+      const CLUSTER_SECRET = shellExec("od -vN 32 -An -tx1 /dev/urandom | tr -d ' \\n'", {
+        stdout: true,
+      }).trim();
+
+      const tmpDir = '/tmp/ipfs-cluster-identity';
+      shellExec(`rm -rf ${tmpDir} && mkdir -p ${tmpDir}`);
+      shellExec(`docker run --rm -v ${tmpDir}:/data/ipfs-cluster ipfs/ipfs-cluster init -f`);
+      const IDENTITY_JSON = JSON.parse(shellExec(`cat ${tmpDir}/identity.json`, { stdout: true }).trim());
+      shellExec(`rm -rf ${tmpDir}`);
+
+      fs.ensureDirSync(privateDir);
+      fs.writeFileSync(secretPath, CLUSTER_SECRET, { mode: 0o600 });
+      fs.writeFileSync(identityPath, JSON.stringify(IDENTITY_JSON, null, 2), { mode: 0o600 });
+
+      logger.info(`IPFS cluster credentials saved (peer ID: ${IDENTITY_JSON.id})`);
+
+      return { CLUSTER_SECRET, IDENTITY_JSON };
+    },
+
+    /**
+     * @method teardown
+     * @description Deletes the existing ipfs-cluster StatefulSet, its Kubernetes Secret,
+     * env ConfigMap, and all PVCs so the next deployment initialises a clean data volume
+     * (ensuring the correct datastore profile is applied by the init container).
+     * @param {object} options
+     * @param {string} options.namespace - Kubernetes namespace.
+     * @param {number} ipfsReplicas - Number of replicas whose PVCs must be removed.
+     * @memberof UnderpostIPFS
+     */
+    teardown(options, ipfsReplicas) {
+      logger.info(`Tearing down existing ipfs-cluster deployment in namespace '${options.namespace}'`);
+      shellExec(`kubectl delete statefulset ipfs-cluster -n ${options.namespace} --ignore-not-found`);
+      shellExec(`kubectl delete secret ipfs-cluster-secret -n ${options.namespace} --ignore-not-found`);
+      shellExec(`kubectl delete configmap env-config -n ${options.namespace} --ignore-not-found`);
+      for (let i = 0; i < ipfsReplicas; i++) {
+        shellExec(
+          `kubectl delete pvc cluster-storage-ipfs-cluster-${i} ipfs-storage-ipfs-cluster-${i} -n ${options.namespace} --ignore-not-found`,
+        );
+      }
+    },
+
+    /**
+     * @method applySecrets
+     * @description Creates (or idempotently updates) the Kubernetes Secret and env ConfigMap
+     * that the StatefulSet pods read at startup.
+     * - Secret `ipfs-cluster-secret`: cluster-secret + bootstrap-peer-priv-key
+     * - ConfigMap `env-config`: bootstrap-peer-id + CLUSTER_SVC_NAME
+     * @param {{ CLUSTER_SECRET: string, IDENTITY_JSON: { id: string, private_key: string } }} credentials
+     * @param {object} options
+     * @param {string} options.namespace - Kubernetes namespace.
+     * @memberof UnderpostIPFS
+     */
+    applySecrets({ CLUSTER_SECRET, IDENTITY_JSON }, options) {
+      logger.info('Applying IPFS cluster Kubernetes Secret and env ConfigMap');
+
+      shellExec(
+        `kubectl create secret generic ipfs-cluster-secret \
+--from-literal=cluster-secret=${CLUSTER_SECRET} \
+--from-literal=bootstrap-peer-priv-key=${IDENTITY_JSON.private_key} \
+--dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
+      );
+
+      shellExec(
+        `kubectl create configmap env-config \
+--from-literal=bootstrap-peer-id=${IDENTITY_JSON.id} \
+--from-literal=CLUSTER_SVC_NAME=ipfs-cluster \
+--dry-run=client -o yaml | kubectl apply -f - -n ${options.namespace}`,
+      );
+    },
+
+    /**
+     * @method applyManifests
+     * @description Applies host-level sysctl tuning (Kind clusters only), the storage class,
+     * the kustomize manifests, and scales the StatefulSet to the requested replica count.
+     * @param {object} options
+     * @param {string} options.namespace - Kubernetes namespace.
+     * @param {boolean} [options.kubeadm] - Whether the cluster is Kubeadm-based.
+     * @param {boolean} [options.k3s] - Whether the cluster is K3s-based.
+     * @param {string} underpostRoot - Absolute path to the underpost project root.
+     * @param {number} ipfsReplicas - Desired replica count.
+     * @memberof UnderpostIPFS
+     */
+    applyManifests(options, underpostRoot, ipfsReplicas) {
+      // Apply UDP buffer sysctl on every Kind node so QUIC (used by IPFS) can reach the
+      // recommended 7.5 MB buffer size. Kind nodes are containers and do NOT inherit the
+      // host sysctl values, so this must be set via docker exec on each node directly.
+      if (!options.kubeadm && !options.k3s) {
+        logger.info('Applying UDP buffer sysctl on Kind nodes');
+        shellExec(
+          `for node in $(kind get nodes); do docker exec $node sysctl -w net.core.rmem_max=7500000 net.core.wmem_max=7500000; done`,
+        );
+      }
+
+      shellExec(`kubectl apply -f ${underpostRoot}/manifests/ipfs/storage-class.yaml`);
+      shellExec(`kubectl apply -k ${underpostRoot}/manifests/ipfs -n ${options.namespace}`);
+
+      // statefulset.yaml hardcodes replicas: 3 as the ceiling; scale down here if needed.
+      shellExec(`kubectl scale statefulset ipfs-cluster --replicas=${ipfsReplicas} -n ${options.namespace}`);
+    },
+
+    /**
+     * @method deploy
+     * @description Full orchestration of the ipfs-cluster StatefulSet deployment:
+     * optionally pulls images, resolves or generates credentials, tears down any existing
+     * deployment, applies secrets, applies manifests, and waits for all pods to be Running.
+     * @param {object} options - Cluster init options forwarded from UnderpostCluster.API.init.
+     * @param {string} options.namespace - Kubernetes namespace.
+     * @param {boolean} [options.pullImage] - Whether to pull container images first.
+     * @param {boolean} [options.kubeadm] - Whether the cluster is Kubeadm-based.
+     * @param {boolean} [options.k3s] - Whether the cluster is K3s-based.
+     * @param {string|number} [options.replicas] - Override replica count (defaults to 3).
+     * @param {string} underpostRoot - Absolute path to the underpost project root.
+     * @memberof UnderpostIPFS
+     */
+    async deploy(options, underpostRoot) {
+      if (options.pullImage === true) {
+        Underpost.cluster.pullImage('ipfs/kubo:latest', options);
+        Underpost.cluster.pullImage('ipfs/ipfs-cluster:latest', options);
+      }
+
+      const credentials = Underpost.ipfs.resolveCredentials(`${underpostRoot}/engine-private`);
+
+      const ipfsReplicas = options.replicas ? parseInt(options.replicas) : 3;
+
+      Underpost.ipfs.teardown(options, ipfsReplicas);
+      Underpost.ipfs.applySecrets(credentials, options);
+      Underpost.ipfs.applyManifests(options, underpostRoot, ipfsReplicas);
+
+      logger.info(`Waiting for ${ipfsReplicas} ipfs-cluster pod(s) to reach Running state`);
+      for (let i = 0; i < ipfsReplicas; i++) {
+        await Underpost.test.statusMonitor(`ipfs-cluster-${i}`, 'Running', 'pods', 1000, 60 * 15);
+      }
+    },
+  };
+}
+
+export default UnderpostIPFS;