@highstate/common 0.9.15 → 0.9.18

package/src/shared/index.ts

@@ -3,3 +3,4 @@ export * from "./dns"
 export * from "./passwords"
 export * from "./ssh"
 export * from "./network"
+export * from "./files"

package/src/shared/network.ts

@@ -19,6 +19,13 @@ export type InputL3Endpoint = network.L3Endpoint | string
  */
 export type InputL4Endpoint = network.L4Endpoint | string
 
+/**
+ * The L7 endpoint for some service.
+ *
+ * The format is: `appProtocol://endpoint[:port][/resource]`
+ */
+export type InputL7Endpoint = network.L7Endpoint | string
+
 /**
  * Stringifies a L3 endpoint object into a string.
  *
@@ -40,7 +47,6 @@ export function l3EndpointToString(l3Endpoint: network.L3Endpoint): string {
  * Stringifies a L4 endpoint object into a string.
  *
  * @param l4Endpoint The L4 endpoint object to stringify.
- *
  * @returns The string representation of the L4 endpoint.
  */
 export function l4EndpointToString(l4Endpoint: network.L4Endpoint): string {
@@ -51,6 +57,37 @@ export function l4EndpointToString(l4Endpoint: network.L4Endpoint): string {
   return `${l3EndpointToString(l4Endpoint)}:${l4Endpoint.port}`
 }
 
+/**
+ * Stringifies a L4 endpoint object into a string with protocol.
+ *
+ * @param l4Endpoint The L4 endpoint object to stringify.
+ * @returns The string representation of the L4 endpoint with protocol.
+ */
+export function l4EndpointWithProtocolToString(l4Endpoint: network.L4Endpoint): string {
+  const protocol = `${l4Endpoint.protocol}://`
+
+  return `${protocol}${l4EndpointToString(l4Endpoint)}`
+}
+
+/**
+ * Stringifies a L7 endpoint object into a string.
+ *
+ * The format is: `appProtocol://endpoint[:port][/resource]`
+ * @param l7Endpoint The L7 endpoint object to stringify.
+ * @returns The string representation of the L7 endpoint.
+ */
+export function l7EndpointToString(l7Endpoint: network.L7Endpoint): string {
+  const protocol = `${l7Endpoint.appProtocol}://`
+
+  let endpoint = l4EndpointToString(l7Endpoint)
+
+  if (l7Endpoint.resource) {
+    endpoint += `/${l7Endpoint.resource}`
+  }
+
+  return `${protocol}${endpoint}`
+}
+
 /**
  * Stringifies a L3 or L4 endpoint object into a string.
  *
@@ -68,6 +105,9 @@ export function l34EndpointToString(l34Endpoint: network.L34Endpoint): string {
 const L34_ENDPOINT_RE =
   /^(?:(?<protocol>[a-z]+):\/\/)?(?:(?:\[?(?<ipv6>[0-9A-Fa-f:]+)\]?)|(?<ipv4>(?:\d{1,3}\.){3}\d{1,3})|(?<hostname>[a-zA-Z0-9-*]+(?:\.[a-zA-Z0-9-*]+)*))(?::(?<port>\d{1,5}))?$/
 
+const L7_ENDPOINT_RE =
+  /^(?<appProtocol>[a-z]+):\/\/(?:(?:\[?(?<ipv6>[0-9A-Fa-f:]+)\]?)|(?<ipv4>(?:\d{1,3}\.){3}\d{1,3})|(?<hostname>[a-zA-Z0-9-*]+(?:\.[a-zA-Z0-9-*]+)*))(?::(?<port>\d{1,5}))?(?:\/(?<resource>.*))?$/
+
 /**
  * Parses a L3 or L4 endpoint from a string.
  *
@@ -208,14 +248,14 @@ export async function requireInputL4Endpoint(
 }
 
 /**
- * Convers L3 endpoint to L4 endpoint by adding a port and protocol.
+ * Converts L3 endpoint to L4 endpoint by adding a port and protocol.
  *
  * @param l3Endpoint The L3 endpoint to convert.
  * @param port The port to add to the L3 endpoint.
  * @param protocol The protocol to add to the L3 endpoint. Defaults to "tcp".
  * @returns The L4 endpoint with the port and protocol added.
  */
-export function l3ToL4Endpoint(
+export function l3EndpointToL4(
   l3Endpoint: InputL3Endpoint,
   port: number,
   protocol: network.L4Protocol = "tcp",
@@ -275,6 +315,53 @@ export function l3EndpointToCidr(l3Endpoint: network.L3Endpoint): string {
   }
 }
 
+const udpAppProtocols = ["dns", "dhcp"]
+
+/**
+ * Parses a L7 endpoint from a string.
+ *
+ * The format is: `appProtocol://endpoint[:port][/resource]`
+ *
+ * @param l7Endpoint The L7 endpoint string to parse.
+ * @returns The parsed L7 endpoint object.
+ */
+export function parseL7Endpoint(l7Endpoint: InputL7Endpoint): network.L7Endpoint {
+  if (typeof l7Endpoint === "object") {
+    return l7Endpoint
+  }
+
+  const match = l7Endpoint.match(L7_ENDPOINT_RE)
+  if (!match) {
+    throw new Error(`Invalid L7 endpoint: "${l7Endpoint}"`)
+  }
+
+  const { appProtocol, ipv6, ipv4, hostname, port, resource } = match.groups!
+
+  let visibility: network.EndpointVisibility = "public"
+
+  if (ipv4 && IPV4_PRIVATE_REGEX.test(ipv4)) {
+    visibility = "external"
+  } else if (ipv6 && IPV6_PRIVATE_REGEX.test(ipv6)) {
+    visibility = "external"
+  }
+
+  return {
+    type: ipv6 ? "ipv6" : ipv4 ? "ipv4" : "hostname",
+    visibility,
+    address: ipv6 || ipv4,
+    hostname: hostname,
+
+    // Default port for L7 endpoints (TODO: add more specific defaults for common protocols)
+    port: port ? parseInt(port, 10) : 443,
+
+    // L7 endpoints typically use TCP, but can also use UDP for specific protocols
+    protocol: udpAppProtocols.includes(appProtocol) ? "udp" : "tcp",
+
+    appProtocol,
+    resource: resource || "",
+  } as network.L7Endpoint
+}
+
 /**
  * Updates the endpoints based on the given mode.
  *

package/src/shared/passwords.ts

@@ -1,6 +1,42 @@
-import { randomBytes } from "@noble/hashes/utils"
+import { bytesToHex, randomBytes } from "@noble/hashes/utils"
 import { secureMask } from "micro-key-producer/password.js"
 
-export function generatePassword() {
+/**
+ * Generates a secure random password strong enough for online use.
+ *
+ * It uses "Safari Keychain Secure Password" format.
+ *
+ * The approximate entropy is 71 bits (https://support.apple.com/guide/security/automatic-strong-passwords-secc84c811c4/web).
+ */
+export function generatePassword(): string {
   return secureMask.apply(randomBytes(32)).password
 }
+
+type KeyFormatMap = {
+  raw: Uint8Array
+  hex: string
+  base64: string
+}
+
+/**
+ * Generates a secure random key strong enough for online use such as encryption.
+ *
+ * The strong entropy is 256 bits.
+ *
+ * @param format The format of the generated key. By default, it is "hex".
+ */
+export function generateKey<TFormat extends keyof KeyFormatMap>(
+  format: TFormat = "hex" as TFormat,
+): KeyFormatMap[TFormat] {
+  const bytes = randomBytes(32)
+
+  if (format === "raw") {
+    return bytes as KeyFormatMap[TFormat]
+  }
+
+  if (format === "base64") {
+    return Buffer.from(bytes).toString("base64") as KeyFormatMap[TFormat]
+  }
+
+  return bytesToHex(bytes) as KeyFormatMap[TFormat]
+}

package/src/shared/ssh.ts

@@ -1,17 +1,20 @@
 import type { common, network, ssh } from "@highstate/library"
 import {
-  getOrCreateSecret,
+  ensureSecretValue,
   output,
   Output,
   secret,
+  toPromise,
   type Input,
   type InstanceTerminal,
+  type UnitSecret,
 } from "@highstate/pulumi"
 import getKeys, { PrivateExport } from "micro-key-producer/ssh.js"
 import { randomBytes } from "micro-key-producer/utils.js"
-import { local, remote } from "@pulumi/command"
+import { remote } from "@pulumi/command"
 import * as images from "../../assets/images.json"
-import { l3EndpointToString, l3ToL4Endpoint } from "./network"
+import { l3EndpointToString, l3EndpointToL4 } from "./network"
+import { Command } from "./command"
 
 export function createSshTerminal(
   credentials: Input<ssh.Credentials | undefined>,
@@ -40,134 +43,299 @@ export function createSshTerminal(
 
     return {
       name: "ssh",
-      title: "Shell",
-      description: "Connect to the server via SSH",
-      icon: "gg:remote",
 
-      image: images["terminal-ssh"].image,
-      command,
+      meta: {
+        title: "Shell",
+        description: "Connect to the server via SSH",
+        icon: "gg:remote",
+      },
 
-      files: {
-        "/password": credentials.password,
+      spec: {
+        image: images["terminal-ssh"].image,
+        command,
 
-        "/private_key": {
-          content: credentials.keyPair?.privateKey,
-          mode: 0o600,
-        },
+        files: {
+          "/password": credentials.password,
 
-        "/known_hosts": {
-          content: `${l3EndpointToString(endpoint)} ${credentials.hostKey}`,
-          mode: 0o644,
+          "/private_key": credentials.keyPair?.privateKey && {
+            content: {
+              type: "embedded",
+              value: credentials.keyPair?.privateKey,
+            },
+            meta: {
+              name: "private_key",
+              mode: 0o600,
+            },
+          },
+
+          "/known_hosts": {
+            content: {
+              type: "embedded",
+              value: `${l3EndpointToString(endpoint)} ${credentials.hostKey}`,
+            },
+            meta: {
+              name: "known_hosts",
+              mode: 0o644,
+            },
+          },
         },
       },
     } satisfies InstanceTerminal
   })
 }
 
-export function generatePrivateKey(): string {
+/**
+ * Generates a secure random SSH private key.
+ * The key is generated using the Ed25519 algorithm.
+ *
+ * @returns The generated SSH private key in PEM format.
+ */
+export function generateSshPrivateKey(): string {
   const seed = randomBytes(32)
 
   return getKeys(seed).privateKey
 }
 
-export function privateKeyToKeyPair(privateKeyString: Input<string>): Output<ssh.KeyPair> {
+/**
+ * Converts a private SSH key string to a KeyPair object.
+ *
+ * @param privateKeyString The private key string to convert.
+ * @returns An Output of the KeyPair object.
+ */
+export function sshPrivateKeyToKeyPair(privateKeyString: Input<string>): Output<ssh.KeyPair> {
   return output(privateKeyString).apply(privateKeyString => {
     const privateKeyStruct = PrivateExport.decode(privateKeyString)
 
     // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
     const privKey = privateKeyStruct.keys[0].privKey.privKey as Uint8Array
 
-    const { fingerprint, publicKey, privateKey } = getKeys(privKey.slice(0, 32))
+    const { fingerprint, publicKey } = getKeys(privKey.slice(0, 32))
 
     return output({
       type: "ed25519" as const,
       fingerprint,
       publicKey,
-      privateKey: secret(privateKey),
+      privateKey: secret(privateKeyString),
     })
   })
 }
 
-export type SshKeyPairInputs = {
-  sshKeyPair?: Input<ssh.KeyPair>
+/**
+ * Ensures that the provided SSH key is available and generates a new key pair if not.
+ * If an existing key pair is provided, it will be used instead of generating a new one.
+ *
+ * @param privateKey The secret containing the private key.
+ * @param existingKeyPair An optional existing key pair to use if available.
+ * @returns An Output of the KeyPair object.
+ */
+export function ensureSshKeyPair(
+  privateKey: UnitSecret<string | undefined>,
+  existingKeyPair?: Input<ssh.KeyPair>,
+): Output<ssh.KeyPair> {
+  if (existingKeyPair) {
+    return output(existingKeyPair)
+  }
+
+  return ensureSecretValue(privateKey, generateSshPrivateKey).value.apply(sshPrivateKeyToKeyPair)
 }
 
-export type SshKeyPairSecrets = {
+export type ServerOptions = {
+  /**
+   * The local name of the server to namespace resources.
+   */
+  name: string
+
+  /**
+   * The fallback hostname to use if the server cannot be determined.
+   *
+   * If not provided, the `name` will be used as the fallback hostname.
+   */
+  fallbackHostname?: string
+
+  /**
+   * The L3 endpoints of the server.
+   */
+  endpoints: network.L3Endpoint[]
+
+  /**
+   * The L4 endpoint of the server.
+   *
+   * If not provided, the first endpoint will be used + `sshPort`.
+   */
+  sshEndpoint?: network.L4Endpoint
+
+  /**
+   * The SSH port to use for the server.
+   *
+   * Will be ignored if `sshEndpoint` is provided.
+   */
+  sshPort?: number
+
+  /**
+   * The SSH user to use for the server.
+   */
+  sshUser?: string
+
+  /**
+   * The SSH password to use for the server.
+   */
+  sshPassword?: Input<string | undefined>
+
+  /**
+   * The SSH private key to use for the server.
+   */
   sshPrivateKey?: Input<string>
-}
 
-export function getOrCreateSshKeyPair(
-  inputs: SshKeyPairInputs,
-  secrets: Output<SshKeyPairSecrets>,
-): Output<ssh.KeyPair> {
-  if (inputs.sshKeyPair) {
-    return output(inputs.sshKeyPair)
-  }
+  /**
+   * Whether the server is expected to have SSH access.
+   * If false, the server will be created without SSH credentials.
+   *
+   * By default, this is true.
+   */
+  hasSsh?: boolean
+
+  /**
+   * Whether to wait for the server to respond to a ping command before returning.
+   *
+   * If true, the command will wait for a successful ping response before proceeding.
+   *
+   * By default, this is equal to `!waitForSsh`, so when `waitForSsh` is true, no extra ping is performed.
+   */
+  waitForPing?: boolean
+
+  /**
+   * The interval in seconds to wait between ping attempts.
+   *
+   * Only used if `waitForPing` is true.
+   * By default, it will wait 5 seconds between attempts.
+   */
+  pingInterval?: number
+
+  /**
+   * The timeout in seconds to wait for the server to respond to a ping command.
+   *
+   * Only used if `waitForPing` is true.
+   * By default, it will wait 5 minutes (300 seconds) before timing out.
+   */
+  pingTimeout?: number
+
+  /**
+   * Whether to wait for the SSH service to be available before returning.
+   *
+   * If true, the command will wait for the SSH service to respond before proceeding.
+   *
+   * By default, this is true if `hasSsh` is true.
+   */
+  waitForSsh?: boolean
 
-  const privateKey = getOrCreateSecret(secrets, "sshPrivateKey", generatePrivateKey)
+  /**
+   * The interval in seconds to wait between SSH connection attempts.
+   *
+   * Only used if `waitForSsh` is true.
+   * By default, it will wait 5 seconds between attempts.
+   */
+  sshCheckInterval?: number
 
-  return privateKey.apply(privateKeyToKeyPair)
+  /**
+   * The timeout in seconds to wait for the SSH service to respond.
+   *
+   * Only used if `waitForSsh` is true.
+   * By default, it will wait 5 minutes (300 seconds) before timing out.
+   */
+  sshCheckTimeout?: number
 }
 
-export function createServerEntity(
-  fallbackHostname: string,
-  endpoint: network.L3Endpoint,
+/**
+ * Creates a server entity with the provided options.
+ * It will create a command to check the SSH service and return the server entity.
+ *
+ * @param options The options for creating the server entity.
+ * @returns A promise that resolves to the created server entity.
+ */
+export async function createServerEntity({
+  name,
+  fallbackHostname,
+  endpoints,
+  sshEndpoint,
   sshPort = 22,
   sshUser = "root",
-  sshPassword?: Input<string | undefined>,
-  sshPrivateKey?: Input<string>,
+  sshPassword,
+  sshPrivateKey,
   hasSsh = true,
-): Output<common.Server> {
+  pingInterval,
+  pingTimeout,
+  waitForPing,
+  waitForSsh,
+  sshCheckInterval,
+  sshCheckTimeout,
+}: ServerOptions): Promise<common.Server> {
+  if (endpoints.length === 0) {
+    throw new Error("At least one L3 endpoint is required to create a server entity")
+  }
+
+  fallbackHostname ??= name
+  waitForSsh ??= hasSsh
+  waitForPing ??= !waitForSsh
+
+  if (waitForPing) {
+    await Command.waitFor(`${name}.ping`, {
+      host: "local",
+      create: `ping -c 1 ${l3EndpointToString(endpoints[0])}`,
+      timeout: pingTimeout ?? 300,
+      interval: pingInterval ?? 5,
+      triggers: [Date.now()],
+    }).wait()
+  }
+
+  if (!hasSsh) {
+    return {
+      hostname: name,
+      endpoints,
+    }
+  }
+
+  sshEndpoint ??= l3EndpointToL4(endpoints[0], sshPort)
+
+  if (waitForSsh) {
+    await Command.waitFor(`${name}.ssh`, {
+      host: "local",
+      create: `nc -zv ${l3EndpointToString(sshEndpoint)} ${sshPort}`,
+      timeout: sshCheckTimeout ?? 300,
+      interval: sshCheckInterval ?? 5,
+      triggers: [Date.now()],
+    }).wait()
+  }
+
   const connection = output({
-    host: l3EndpointToString(endpoint),
-    port: sshPort,
+    host: l3EndpointToString(sshEndpoint),
+    port: sshEndpoint.port,
     user: sshUser,
     password: sshPassword,
     privateKey: sshPrivateKey,
     dialErrorLimit: 3,
   })
 
-  if (!hasSsh) {
-    return output({
-      hostname: fallbackHostname,
-      endpoints: [endpoint],
-    })
-  }
-
-  const command = new local.Command("check-ssh", {
-    create: `nc -zv ${l3EndpointToString(endpoint)} ${sshPort} && echo "up" || echo "down"`,
+  const hostnameResult = new remote.Command("hostname", {
+    connection,
+    create: "hostname",
+    triggers: [Date.now()],
   })
 
-  return command.stdout.apply(result => {
-    if (result === "down") {
-      return output({
-        hostname: fallbackHostname,
-        endpoints: [endpoint],
-      })
-    }
-
-    const hostnameResult = new remote.Command("hostname", {
-      connection,
-      create: "hostname",
-      triggers: [Date.now()],
-    })
-
-    const hostKeyResult = new remote.Command("host-key", {
-      connection,
-      create: "cat /etc/ssh/ssh_host_ed25519_key.pub",
-      triggers: [Date.now()],
-    })
+  const hostKeyResult = new remote.Command("host-key", {
+    connection,
+    create: "cat /etc/ssh/ssh_host_ed25519_key.pub",
+    triggers: [Date.now()],
+  })
 
-    return output({
-      endpoints: [endpoint],
-      hostname: hostnameResult.stdout.apply(x => x.trim()),
-      ssh: {
-        endpoints: [l3ToL4Endpoint(endpoint, sshPort)],
-        user: sshUser,
-        hostKey: hostKeyResult.stdout.apply(x => x.trim()),
-        password: sshPassword,
-        keyPair: sshPrivateKey ? privateKeyToKeyPair(sshPrivateKey) : undefined,
-      },
-    })
+  return await toPromise({
+    endpoints,
+    hostname: hostnameResult.stdout.apply(x => x.trim()),
+    ssh: {
+      endpoints: [sshEndpoint],
+      user: sshUser,
+      hostKey: hostKeyResult.stdout.apply(x => x.trim()),
+      password: sshPassword,
+      keyPair: sshPrivateKey ? sshPrivateKeyToKeyPair(sshPrivateKey) : undefined,
+    },
   })
 }

package/src/units/existing-server/index.ts

@@ -1,8 +1,9 @@
 import { common } from "@highstate/library"
-import { forUnit, toPromise } from "@highstate/pulumi"
+import { forUnit } from "@highstate/pulumi"
 import {
   createServerEntity,
   createSshTerminal,
+  ensureSshKeyPair,
   l3EndpointToString,
   requireInputL3Endpoint,
 } from "../../shared"
@@ -10,24 +11,24 @@ import {
 const { name, args, inputs, secrets, outputs } = forUnit(common.existingServer)
 
 const endpoint = await requireInputL3Endpoint(args.endpoint, inputs.endpoint)
-const privateKey = await toPromise(inputs.sshKeyPair?.privateKey ?? secrets.sshPrivateKey)
+const keyPair = ensureSshKeyPair(secrets.sshPrivateKey, inputs.sshKeyPair)
 
-const server = createServerEntity(
+const server = await createServerEntity({
   name,
-  endpoint,
-  args.sshPort,
-  args.sshUser,
-  secrets.sshPassword,
-  privateKey,
-)
+  endpoints: [endpoint],
+  sshPort: args.sshPort,
+  sshUser: args.sshUser,
+  sshPassword: secrets.sshPassword.value,
+  sshPrivateKey: keyPair.privateKey,
+})
 
 export default outputs({
   server,
   endpoints: server.endpoints,
 
-  $status: {
+  $statusFields: {
     hostname: server.hostname,
-    endpoints: server.endpoints.apply(endpoints => endpoints.map(l3EndpointToString)),
+    endpoints: server.endpoints.map(l3EndpointToString),
   },
 
   $terminals: [createSshTerminal(server.ssh)],

package/src/units/server-dns/index.ts

@@ -20,7 +20,7 @@ export default outputs({
 
   endpoints,
 
-  $status: {
+  $statusFields: {
     endpoints: endpoints.map(l3EndpointToString),
   },
 })

package/src/units/server-patch/index.ts

@@ -19,7 +19,7 @@ export default outputs({
 
   endpoints,
 
-  $status: {
+  $statusFields: {
     endpoints: endpoints.map(l3EndpointToString),
   },
 })

package/src/units/ssh/key-pair/index.ts

@@ -1,15 +1,24 @@
 import { ssh } from "@highstate/library"
-import { forUnit, getOrCreateSecret } from "@highstate/pulumi"
-import { generatePrivateKey, privateKeyToKeyPair } from "../../../shared"
+import { forUnit } from "@highstate/pulumi"
+import { ensureSshKeyPair } from "../../../shared"
 
-const { secrets, outputs } = forUnit(ssh.keyPair)
+const { name, secrets, outputs } = forUnit(ssh.keyPair)
 
-const privateKey = getOrCreateSecret(secrets, "privateKey", generatePrivateKey)
-const keyPair = privateKeyToKeyPair(privateKey)
+const keyPair = ensureSshKeyPair(secrets.privateKey)
 
 export default outputs({
-  keyPair: privateKeyToKeyPair(privateKey),
-  $status: {
+  keyPair,
+  publicKeyFile: {
+    meta: {
+      name: `${name}.pub`,
+      mode: 0o644,
+    },
+    content: {
+      type: "embedded",
+      value: keyPair.publicKey,
+    },
+  },
+  $statusFields: {
     fingerprint: keyPair.fingerprint,
     publicKey: keyPair.publicKey,
   },