@metaplay/metaplay-auth 1.3.0 → 1.4.1

package/src/deployment.ts

@@ -1,13 +1,13 @@
-import { existsSync } from 'fs'
 import { KubeConfig, CoreV1Api, V1Pod } from '@kubernetes/client-node'
 import { logger } from './logging.js'
 import type { CoreV1ApiListNamespacedPodRequest, CoreV1ApiReadNamespacedPodLogRequest } from '@kubernetes/client-node'
 
 enum GameServerPodPhase {
-  Pending = 'Pending', // Still being deployed
   Ready = 'Ready', // Successfully started and ready to accept traffic
-  Failed = 'Failed', // Failed to start
+  Running = 'Running', // Started but not yet serving traffic (eg, waiting for peers to connect)
+  Pending = 'Pending', // Still being deployed
   Unknown = 'Unknown', // Unknown status -- treat like Pending
+  Failed = 'Failed', // Failed to start
 }
 
 interface GameServerPodStatus {
@@ -18,6 +18,7 @@ interface GameServerPodStatus {
 
 async function fetchGameServerPods (k8sApi: CoreV1Api, namespace: string) {
   // Define label selector for gameserver
+  logger.debug(`Fetching game server pods from Kubernetes: namespace=${namespace}`)
   const param: CoreV1ApiListNamespacedPodRequest = {
     namespace,
     labelSelector: 'app=metaplay-server'
@@ -42,14 +43,14 @@ function resolvePodContainersConditions (pod: V1Pod): GameServerPodStatus {
     return { phase: GameServerPodPhase.Unknown, message: 'Unable to determine pod container statuses: pod.status.containerStatuses is empty' }
   }
 
-  // Only one container allowed in the game server pod
-  if (containerStatuses.length !== 1) {
-    throw new Error(`Internal error: Expecting only one container in the game server pod, got ${containerStatuses.length}`)
+  // Find the shard-server container from the pod (ignore others)
+  const containerStatus = containerStatuses.find(status => status.name === 'shard-server')
+  if (!containerStatus) {
+    return { phase: GameServerPodPhase.Unknown, message: 'Unable to find container shard-server from the pod' }
   }
 
   // Handle missing container state
-  const containerStatus = containerStatuses[0]
-  console.log('Container status:', containerStatus)
+  logger.debug(`Container status for pod ${pod.metadata?.name ?? '<unnamed>'}: ${JSON.stringify(containerStatus, undefined, 2)}`)
   const containerState = containerStatus.state
   if (!containerState) {
     return { phase: GameServerPodPhase.Unknown, message: 'Unable to get container state' }
@@ -57,8 +58,12 @@ function resolvePodContainersConditions (pod: V1Pod): GameServerPodStatus {
 
   // Check if container running & ready
   const containerName = containerStatus.name
-  if (containerStatus.ready && containerState.running) {
-    return { phase: GameServerPodPhase.Ready, message: `Container ${containerName} is in ready phase and was started at ${containerState.running.startedAt}`, details: containerState.running }
+  if (containerState.running) {
+    if (containerStatus.ready) {
+      return { phase: GameServerPodPhase.Ready, message: `Container ${containerName} is in ready phase`, details: containerState.running }
+    } else {
+      return { phase: GameServerPodPhase.Running, message: `Container ${containerName} is in running phase`, details: containerState.running }
+    }
   }
 
   // \note these may not be complete (or completely accurate)
@@ -94,7 +99,7 @@ function resolvePodContainersConditions (pod: V1Pod): GameServerPodStatus {
     // \todo handle containerState.terminated states (what do these even mean)
   }
 
-  console.log('Game server pod container in unknown state:', containerState)
+  logger.debug('Game server pod container in unknown state:', containerState)
   return { phase: GameServerPodPhase.Unknown, message: 'Container in unknown state', details: containerState }
 }
 
@@ -137,7 +142,11 @@ function resolvePodStatusConditions (pod: V1Pod): GameServerPodStatus {
 }
 
 function resolvePodStatus (pod: V1Pod): GameServerPodStatus {
-  // console.log('Pod.status:', JSON.stringify(pod.status, undefined, 2))
+  // logger.debug('resolvePodStatus(): pod =', JSON.stringify(pod, undefined, 2))
+
+  if (!pod) {
+    return { phase: GameServerPodPhase.Unknown, message: 'Received empty pod from Kubernetes' }
+  }
 
   if (!pod.status) {
     return { phase: GameServerPodPhase.Unknown, message: 'Unable to access pod.status from Kubernetes' }
@@ -163,12 +172,12 @@ function resolvePodStatus (pod: V1Pod): GameServerPodStatus {
 }
 
 async function fetchPodLogs (k8sApi: CoreV1Api, pod: V1Pod) {
-  console.log('Fetching logs for pod..')
   const podName = pod.metadata?.name
+  logger.debug(`Fetching logs for pod '${podName}'..`)
   const namespace = pod.metadata?.namespace
   const containerName = pod.spec?.containers[0].name // \todo Handle multiple containers?
   if (!podName || !namespace || !containerName) {
-    throw new Error('Unable to determine pod metadata')
+    throw new Error('Unable to determine pod and container metadata')
   }
 
   const params: CoreV1ApiReadNamespacedPodLogRequest = {
@@ -196,14 +205,15 @@ async function checkGameServerPod (k8sApi: CoreV1Api, pod: V1Pod) {
 
   // Classify game server status
   const podStatus = resolvePodStatus(pod)
+  const podName = pod.metadata?.name ?? '<unable to resolve name>'
 
   // If game server launch failed, get the error logs
   if (podStatus.phase === GameServerPodPhase.Failed) {
     const logs = await fetchPodLogs(k8sApi, pod)
-    console.log('Pod logs:\n' + logs)
+    console.log(`Logs from pod '${podName}':\n${logs}`)
   }
 
-  console.log(`Pod ${pod.metadata?.name} status:`, podStatus)
+  logger.debug(`Pod ${podName} status: ${JSON.stringify(podStatus, undefined, 2)}`)
   return podStatus
 }
 
@@ -211,29 +221,16 @@ async function delay (ms: number): Promise<void> {
   return await new Promise<void>(resolve => setTimeout(resolve, ms))
 }
 
-export async function checkGameServerDeployment (namespace: string): Promise<number> {
-  logger.info(`Validating game server deployment in namespace ${namespace}`)
+function anyPodsInPhase (podStatuses: GameServerPodStatus[], phase: GameServerPodPhase): boolean {
+  return podStatuses.some(status => status.phase === phase)
+}
 
-  // Check that the KUBECONFIG environment variable exists
-  const kubeconfigPath = process.env.KUBECONFIG
-  if (!kubeconfigPath) {
-    throw new Error('The KUBECONFIG environment variable must be specified')
-  }
+function allPodsInPhase (podStatuses: GameServerPodStatus[], phase: GameServerPodPhase): boolean {
+  return podStatuses.every(status => status.phase === phase)
+}
 
-  // Check that the kubeconfig file exists
-  if (!await existsSync(kubeconfigPath)) {
-    throw new Error(`The environment variable KUBECONFIG points to a file '${kubeconfigPath}' that doesn't exist`)
-  }
-
-  // Create Kubernetes API instance (with default kubeconfig)
-  const kc = new KubeConfig()
-  // Loda kubeconfig from file and throw error if validation fails.
-  try {
-    kc.loadFromFile(kubeconfigPath)
-  } catch (error) {
-    throw new Error(`Failed to load or validate kubeconfig. ${error}`)
-  }
-  const k8sApi = kc.makeApiClient(CoreV1Api)
+export async function checkGameServerDeployment (namespace: string, kubeconfig: KubeConfig): Promise<number> {
+  const k8sApi = kubeconfig.makeApiClient(CoreV1Api)
 
   // Figure out when to stop
   const startTime = Date.now()
@@ -242,33 +239,49 @@ export async function checkGameServerDeployment (namespace: string): Promise<num
   while (true) {
     // Check pod states
     const pods = await fetchGameServerPods(k8sApi, namespace)
-    const podStatus = await checkGameServerPod(k8sApi, pods[0]) // \todo Handle all pods
-
-    switch (podStatus.phase) {
-      case GameServerPodPhase.Ready:
-        console.log('Gameserver successfully started')
+    logger.debug(`Found ${pods?.length} pod(s) deployed in Kubernetes`)
+    if (pods.length > 0) {
+      // Resolve status for all pods
+      const podStatuses = await Promise.all(pods.map(async pod => await checkGameServerPod(k8sApi, pod)))
+      logger.debug(`Pod phases: ${podStatuses.map(status => status.phase)}`)
+
+      // Handle state of the deployment
+      if (anyPodsInPhase(podStatuses, GameServerPodPhase.Failed)) {
+        logger.error(`Gameserver start failed with pod statuses: ${JSON.stringify(podStatuses, undefined, 2)}`)
+        console.log('Gameserver failed to start due to the pods not starting properly! See above for details.')
+        for (let ndx = 0; ndx < pods.length; ndx += 1) {
+          const status = podStatuses[ndx]
+          const suffix = (status.phase !== GameServerPodPhase.Ready) ? ` -- ${status.message}` : ''
+          console.log(`  ${pods[ndx].metadata?.name}: ${status.phase}${suffix}`)
+        }
+        return 1
+      } else if (anyPodsInPhase(podStatuses, GameServerPodPhase.Unknown) || anyPodsInPhase(podStatuses, GameServerPodPhase.Pending) || anyPodsInPhase(podStatuses, GameServerPodPhase.Running)) {
+        console.log('Waiting for gameserver(s) to be ready...')
+        for (let ndx = 0; ndx < pods.length; ndx += 1) {
+          const status = podStatuses[ndx]
+          const suffix = (status.phase !== GameServerPodPhase.Ready) ? ` -- ${status.message}` : ''
+          console.log(`  ${pods[ndx].metadata?.name}: ${status.phase}${suffix}`)
+        }
+      } else if (allPodsInPhase(podStatuses, GameServerPodPhase.Ready)) {
+        console.log('Gameserver is up and ready to serve!')
         // \todo add further readiness checks -- ping endpoint, ping dashboard, other checks?
         return 0
-
-      case GameServerPodPhase.Failed:
-        console.log('Gameserver failed to start')
-        return 1
-
-      case GameServerPodPhase.Pending:
-        console.log('Gameserver still starting')
-        break
-
-      case GameServerPodPhase.Unknown:
-      default:
-        console.log('Gameserver in unknown state')
-        break
+      } else {
+        console.log('Deployment in inconsistent state, waiting...')
+        for (let ndx = 0; ndx < pods.length; ndx += 1) {
+          const status = podStatuses[ndx]
+          const suffix = (status.phase !== GameServerPodPhase.Ready) ? ` -- ${status.message}` : ''
+          console.log(`  ${pods[ndx].metadata?.name}: ${status.phase}${suffix}`)
+        }
+      }
     }
 
     if (Date.now() >= timeoutAt) {
-      logger.error('Timeout while waiting for gameserver to initialize')
+      console.log('Deployment failed! Timeout while waiting for gameserver to initialize.')
       return 124 // timeout
     }
 
-    await delay(1000)
+    // Sleep a bit to avoid spamming the log
+    await delay(2000)
   }
 }

package/src/stackapi.ts

@@ -138,13 +138,23 @@ export class StackAPI {
 
     logger.debug(`Getting KubeConfig from ${url}...`)
 
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: {
-        Authorization: `Bearer ${this.accessToken}`,
-        'Content-Type': 'application/json'
+    let response
+    try {
+      response = await fetch(url, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.accessToken}`,
+          'Content-Type': 'application/json'
+        }
+      })
+    } catch (error: any) {
+      logger.error(`Failed to fetch kubeconfig from ${url}`)
+      logger.error('Fetch error details:', error)
+      if (error.cause?.code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
+        throw new Error(`Failed to to fetch kubeconfig: SSL certificate validation failed for ${url}. Is someone trying to tamper with your internet connection?`)
       }
-    })
+      throw new Error(`Failed to fetch kubeconfig from ${url}: ${error}`)
+    }
 
     if (response.status !== 200) {
       throw new Error(`Failed to fetch KubeConfigs: ${response.statusText}`)
@@ -171,20 +181,30 @@ export class StackAPI {
     } else if (gs.organization != null && gs.project != null && gs.environment != null) {
       url = `${this._stackApiBaseUrl}/v1/servers/${gs.organization}/${gs.project}/${gs.environment}/credentials/k8s?type=execcredential`
       gsSlug = `${gs.organization}.${gs.project}.${gs.environment}`
-      execArgs.push('--organization', gs.organization, '--project', gs.project, '--environment', gs.environment)
+      execArgs.push(`${gs.organization}-${gs.project}-${gs.environment}`)
     } else {
       throw new Error('Invalid arguments for getKubeConfigExecCredential')
     }
 
     logger.debug(`Getting Kubernetes KubeConfig from ${url}...`)
 
-    const response = await fetch(url, {
-      method: 'POST',
-      headers: {
-        Authorization: `Bearer ${this.accessToken}`,
-        'Content-Type': 'application/json'
+    let response
+    try {
+      response = await fetch(url, {
+        method: 'POST',
+        headers: {
+          Authorization: `Bearer ${this.accessToken}`,
+          'Content-Type': 'application/json'
+        }
+      })
+    } catch (error: any) {
+      logger.error(`Failed to fetch kubeconfig from ${url}`)
+      logger.error('Fetch error details:', error)
+      if (error.cause?.code === 'UNABLE_TO_VERIFY_LEAF_SIGNATURE') {
+        throw new Error(`Failed to to fetch kubeconfig: SSL certificate validation failed for ${url}. Is someone trying to tamper with your internet connection?`)
       }
-    })
+      throw new Error(`Failed to fetch kubeconfig from ${url}: ${error}`)
+    }
 
     if (response.status !== 200) {
       throw new Error(`Failed to fetch Kubernetes KubeConfig: ${response.statusText}`)

package/src/utils.ts

@@ -1,3 +1,5 @@
+import { spawn } from 'child_process'
+
 export function stackApiBaseFromOptions (arg: string, options: Record<string, any>): string {
   return ''
 }
@@ -46,3 +48,38 @@ export function getGameserverAdminUrl (uri: string): string {
 
   return `${hostname}-admin.${domain}`
 }
+
+interface CommandResult {
+  code: number | null
+  stdout: any[]
+  stderr: any[]
+}
+
+export async function executeCommand (command: string, args: string[]): Promise<CommandResult> {
+  return await new Promise((resolve, reject) => {
+    const childProcess = spawn(command, args) //, { stdio: 'inherit' }) -- this pipes stdout & stderr to our output
+    let stdout: any[] = []
+    let stderr: any[] = []
+
+    childProcess.stdout?.on('data', (data) => {
+      stdout += data
+    })
+
+    childProcess.stderr?.on('data', (data) => {
+      stderr += data
+    })
+
+    // If program runs to completion, resolve promise with success
+    childProcess.on('close', (code) => {
+      resolve({ code, stdout, stderr })
+    })
+
+    childProcess.on('error', (err) => {
+      reject(
+        new Error(
+          `Failed to execute command '${command} ${args.join(' ')}': ${err.message}`
+        )
+      )
+    })
+  })
+}