@ossy/deployment-tools 0.0.45 → 0.0.47

package/src/infrastructure/container-server/caddy.service.js

@@ -0,0 +1,69 @@
+const systemdServiceFile = `
+# caddy-api.service
+#
+# For using Caddy with its API.
+#
+# This unit is "durable" in that it will automatically resume
+# the last active configuration if the service is restarted.
+#
+# See https://caddyserver.com/docs/install for instructions.
+
+[Unit]
+Description=Caddy
+Documentation=https://caddyserver.com/docs/
+After=network.target network-online.target
+Requires=network-online.target
+
+[Service]
+Type=notify
+User=caddy
+Group=caddy
+ExecStart=/usr/bin/caddy.route53 run --environ --resume
+TimeoutStopSec=5s
+LimitNOFILE=1048576
+LimitNPROC=512
+PrivateTmp=true
+ProtectSystem=full
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+
+[Install]
+WantedBy=multi-user.target cloud-init.target
+`
+
+class CaddyService {
+
+  static install() {
+    return [
+      `sudo echo "${systemdServiceFile}" >> /etc/systemd/system/caddy-route53.service`,
+      'sudo apt install -y debian-keyring debian-archive-keyring apt-transport-https',
+      'sudo curl -1sLf \'https://dl.cloudsmith.io/public/caddy/stable/gpg.key\' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-stable-archive-keyring.gpg',
+      'sudo curl -1sLf \'https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt\' | sudo tee /etc/apt/sources.list.d/caddy-stable.list',
+      'sudo curl -1sLf \'https://dl.cloudsmith.io/public/caddy/xcaddy/gpg.key\' | sudo gpg --dearmor -o /usr/share/keyrings/caddy-xcaddy-archive-keyring.gpg',
+      'sudo curl -1sLf \'https://dl.cloudsmith.io/public/caddy/xcaddy/debian.deb.txt\' | sudo tee /etc/apt/sources.list.d/caddy-xcaddy.list',
+      'sudo add-apt-repository ppa:longsleep/golang-backports -y',
+      'sudo apt update',
+      'sudo apt install caddy xcaddy golang-go -y',
+      'sudo xcaddy build --with github.com/caddy-dns/route53',
+      'sudo mv ./caddy /usr/bin/caddy.route53'
+    ]
+  }
+
+  static enable() {
+    return [
+      'sudo systemctl disable caddy.service',
+      'sudo systemctl enable caddy-route53.service'
+    ]
+  }
+
+  static start() {
+    return [
+      'sudo systemctl stop caddy.service',
+      'sudo systemctl start caddy-route53.service'
+    ]
+  }
+
+}
+
+module.exports = {
+  CaddyService
+}

package/src/infrastructure/container-server/container-server.js

@@ -0,0 +1,175 @@
+const { CfnOutput } = require('aws-cdk-lib')
+const { Construct } = require('constructs')
+const {
+  Instance,
+  InstanceType,
+  InstanceClass,
+  InstanceSize,
+  GenericLinuxImage,
+  Vpc,
+  SecurityGroup,
+  Peer,
+  Port,
+  UserData
+} = require('aws-cdk-lib/aws-ec2')
+const { HostedZone, ARecord, RecordTarget } = require('aws-cdk-lib/aws-route53')
+const { Role, ServicePrincipal, Policy, PolicyStatement, Effect } = require('aws-cdk-lib/aws-iam')
+const {
+  getInstallNodeJs,
+  getInstallNpm,
+  getInstallDocker
+} = require('./user-data-commands')
+const { CaddyService } = require('./caddy.service')
+const { DeploymentToolsService } = require('./deployment-tools.service')
+const { AwsProfile } = require('./aws-profile')
+const { SupportedRegions } = require('../../config')
+
+/**
+ * Platform template definition
+ * @namespace ContainerServer
+ * @typedef {Object} ContainerServerProps
+ * @property {PlatformConfig} platformConfig - platform config
+ * @property {Bucket} platformConfigBucket - aws bucket
+ */
+const InstanceImages = {
+  UBUNTU: 'ami-092cce4a19b438926'
+}
+
+class ContainerServer extends Construct {
+
+  /**
+   * EC2 Instance that docker containers are served on
+   *
+   * @param {object} scope - scope
+   * @param {string} id - id
+   * @param {ContainerServerProps} props - ContainerServerProps
+   */
+  constructor(scope, id, props) {
+    super(scope, id)
+
+    const hostedZone = HostedZone.fromLookup(this, 'HostedZone', { domainName: props.platformConfig.domain })
+
+    const vpc = Vpc.fromLookup(this, 'VPC', {
+      isDefault: true
+    })
+
+    const securityGroup = new SecurityGroup(this, 'SecurityGroup', {
+      vpc,
+      allowAllOutbound: true
+    })
+
+    securityGroup.addIngressRule(
+      Peer.anyIpv4(),
+      Port.tcp(22),
+      'allow SSH access from anywhere'
+    )
+
+    securityGroup.addIngressRule(
+      Peer.anyIpv4(),
+      Port.tcp(80),
+      'allow HTTP traffic from anywhere'
+    )
+
+    securityGroup.addIngressRule(
+      Peer.anyIpv4(),
+      Port.tcp(443),
+      'allow HTTPS traffic from anywhere'
+    )
+
+    const role = new Role(this, 'role', {
+      assumedBy: new ServicePrincipal('ec2.amazonaws.com')
+    })
+
+    role.attachInlinePolicy(new Policy(this, 'policy', {
+      statements: [
+        new PolicyStatement({
+          effect: Effect.ALLOW,
+          actions: [
+            'route53:ListResourceRecordSets',
+            'route53:GetChange',
+            'route53:ChangeResourceRecordSets'
+          ],
+          resources: [
+            `arn:aws:route53:::hostedzone/${hostedZone.hostedZoneId}`,
+            'arn:aws:route53:::change/*'
+          ]
+        }),
+        new PolicyStatement({
+          effect: Effect.ALLOW,
+          actions: [
+            'route53:ListHostedZonesByName',
+            'route53:ListHostedZones'
+          ],
+          resources: ['*']
+        })
+      ]
+    }))
+
+    const userData = UserData.forLinux()
+
+    userData.addCommands(
+      'sudo apt update -y',
+      ...getInstallNodeJs(),
+      ...getInstallNpm(),
+      ...getInstallDocker(),
+      'sudo apt-get install awscli --yes',
+      ...AwsProfile.writeFile(role.roleArn, SupportedRegions.North),
+      ...CaddyService.install(),
+      ...DeploymentToolsService.install()
+    )
+
+    userData.addS3DownloadCommand({
+      bucket: props.platformConfigBucket,
+      bucketKey: 'platform-config.json',
+      localFile: '/home/ubuntu/platform-config.json'
+    })
+
+    userData.addCommands(
+      'sudo systemctl daemon-reload',
+      ...CaddyService.enable(),
+      ...DeploymentToolsService.enable(),
+      ...CaddyService.start(),
+      ...DeploymentToolsService.start()
+    )
+
+    const ec2Instance = new Instance(this, 'Ec2Instance', {
+      vpc,
+      securityGroup,
+      userData,
+      role,
+      instanceType: InstanceType.of(
+        InstanceClass.T3,
+        InstanceSize.MICRO
+      ),
+      machineImage: new GenericLinuxImage({
+        [SupportedRegions.North]: InstanceImages.UBUNTU
+      }),
+      keyName: props.platformConfig.awsKeyPairName
+    })
+
+    props.platformConfigBucket.grantRead(ec2Instance, '*')
+
+    // const hostedZone = !!props.createNewHostedZone
+    //   ? new PublicHostedZone(this, 'HostedZone', { zoneName: props.domain })
+    //   : HostedZone.fromLookup(this, 'HostedZone', { domainName: props.domain });
+
+    // eslint-disable-next-line no-new
+    new ARecord(this, 'WildcardRecord', {
+      zone: hostedZone,
+      recordName: `*.${props.platformConfig.activeEnvironment}.${props.platformConfig.domain}`,
+      target: RecordTarget.fromIpAddresses(ec2Instance.instancePublicIp)
+    })
+
+    // eslint-disable-next-line no-new
+    new CfnOutput(this, 'Intance Ip', {
+      value: ec2Instance.instancePublicIp,
+      description: 'Public ip of the ec2 instance',
+      exportName: 'instanceIp'
+    })
+
+  }
+}
+
+module.exports = {
+  ContainerServer
+}

package/src/infrastructure/container-server/deployment-tools.service.js

@@ -0,0 +1,37 @@
+const systemdServiceFile = `
+[Unit]
+Description=D
+After=network.target caddy-route53.service
+
+[Service]
+EnvironmentFile=/etc/environment
+User=caddy
+Group=caddy
+AmbientCapabilities=CAP_NET_BIND_SERVICE
+CapabilityBoundingSet=CAP_NET_BIND_SERVICE
+ExecStart=/usr/bin/npx --yes @ossy/deployment-tools start --platforms /home/ubuntu/deployment-platform.json
+Restart=on-failure
+
+[Install]
+WantedBy=multi-user.target cloud-init.target
+`
+
+class DeploymentToolsService {
+
+  static install() {
+    return [`sudo echo "${systemdServiceFile}" >> /etc/systemd/system/deployment-tools.service`]
+  }
+
+  static enable() {
+    return ['sudo systemctl enable deployment-tools.service']
+  }
+
+  static start() {
+    return ['sudo systemctl start deployment-tools.service']
+  }
+
+}
+
+module.exports = {
+  DeploymentToolsService
+}

package/src/infrastructure/container-server/index.js

@@ -0,0 +1,3 @@
+const { ContainerServer } = require('./container-server')
+
+module.exports = { ContainerServer }

package/src/infrastructure/container-server/user-data-commands.js

@@ -0,0 +1,32 @@
+
+const getInstallDocker = () => [
+  'apt-get remove docker docker-engine docker.io containerd runc',
+  `apt-get install \
+    apt-transport-https \
+    ca-certificates \
+    curl \
+    gnupg-agent \
+    software-properties-common -y`,
+  'curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo apt-key add -',
+  `sudo add-apt-repository \
+     "deb [arch=amd64] https://download.docker.com/linux/ubuntu \
+     $(lsb_release -cs) \
+     stable"`,
+  'sudo apt-get update',
+  'sudo apt-get install docker-ce docker-ce-cli containerd.io -y'
+]
+
+const getInstallNodeJs = () => [
+  'curl -sL https://deb.nodesource.com/setup_16.x | sudo bash -',
+  'sudo apt-get -y install nodejs'
+]
+
+const getInstallNpm = () => [
+  'sudo apt install npm'
+]
+
+module.exports = {
+  getInstallDocker,
+  getInstallNodeJs,
+  getInstallNpm
+}

package/src/infrastructure/establish-trust-stack.js

@@ -0,0 +1,65 @@
+const { Stack, Duration } = require('aws-cdk-lib')
+const {
+  OpenIdConnectProvider,
+  OpenIdConnectPrincipal,
+  Role,
+  Effect,
+  PolicyDocument,
+  PolicyStatement
+} = require('aws-cdk-lib/aws-iam')
+
+class EstablishTrustStack extends Stack {
+  /**
+   * Establishes trust between GithHub and Amazon Web Services.
+   * This is needed so that we can interact with Amazon Web Services through
+   * GitHub Actions without having to manually provide credentials every time we
+   * run a workflow
+   *
+   * @param {object} scope - scope
+   * @param {string} id - id
+   * @param {PlatformTemplate} props - PlatformTemplate
+   */
+  constructor(scope, id, props) {
+    super(scope, id, props)
+
+    const provider = new OpenIdConnectProvider(this, 'GitHubProvider', {
+      url: 'https://token.actions.githubusercontent.com',
+      clientIds: ['sts.amazonaws.com']
+    })
+
+    const GitHubPrincipal = new OpenIdConnectPrincipal(provider)
+      .withConditions({
+        StringLike: {
+          'token.actions.githubusercontent.com:sub':
+            `repo:${props.config.ciGithubActionsRepo}:*`
+        }
+      })
+
+    // eslint-disable-next-line no-new
+    new Role(this, 'GitHubActionsRole', {
+      assumedBy: GitHubPrincipal,
+      description: 'Role assumed by GitHubPrincipal for deploying from CI using aws cdk',
+      roleName: props.config.awsRoleToAssume,
+      maxSessionDuration: Duration.hours(1),
+      inlinePolicies: {
+        CdkDeploymentPolicy: new PolicyDocument({
+          assignSids: true,
+          statements: [
+            new PolicyStatement({
+              effect: Effect.ALLOW,
+              actions: ['sts:AssumeRole'],
+              resources: [
+                `arn:aws:iam::${props.config.awsAccountId}:role/cdk-*`,
+                `arn:aws:sqs:${props.config.awsRegion}:${props.config.awsAccountId}:*`
+              ]
+            })
+          ]
+        })
+      }
+    })
+  }
+}
+
+module.exports = {
+  EstablishTrustStack
+}

package/src/infrastructure/platform-stack.js

@@ -0,0 +1,53 @@
+/* eslint-disable no-new */
+const { Stack, Duration, RemovalPolicy } = require('aws-cdk-lib')
+const { Queue } = require('aws-cdk-lib/aws-sqs')
+const { Bucket, BucketEncryption, BlockPublicAccess } = require('aws-cdk-lib/aws-s3')
+const { Source, BucketDeployment } = require('aws-cdk-lib/aws-s3-deployment')
+
+const { ContainerServer } = require('./container-server')
+
+const { SupportedDeploymentTypes } = require('../config')
+
+class PlatformStack extends Stack {
+  constructor(scope, id, props) {
+    super(scope, id, props)
+
+    if (!props?.config) {
+      throw ('[PlatformStack] No template config provided')
+    }
+
+    const isContainerDeploymentsEnabled = props.config.supportedDeploymentTypes.includes(SupportedDeploymentTypes.Container)
+
+    if (isContainerDeploymentsEnabled) {
+
+      new Queue(this, 'DeploymentQueue', {
+        queueName: `${props.config.platformName}-${props.config.activeEnvironment}`,
+        receiveMessageWaitTime: Duration.seconds(20)
+      })
+
+      const platformConfigBucket = new Bucket(this, 'PlatformConfig', {
+        bucketName: `${props.config.platformName}-${props.config.activeEnvironment}`,
+        blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
+        encryption: BucketEncryption.S3_MANAGED,
+        removalPolicy: RemovalPolicy.DESTROY,
+        autoDeleteObjects: true
+      })
+
+      const platformConfigDeployment = new BucketDeployment(this, 'PlatformConfigDeployment', {
+        sources: [Source.jsonData('platform-config.json', props.config)],
+        destinationBucket: platformConfigBucket
+      })
+
+      new ContainerServer(this, 'ContainerServer', {
+        platformConfig: props.config,
+        platformConfigBucket: platformConfigDeployment.deployedBucket
+      })
+
+    }
+
+  }
+}
+
+module.exports = {
+  PlatformStack
+}

package/src/{caddy-client.js → server/caddy.js}

@@ -1,5 +1,5 @@
 const fetch = require('node-fetch')
-const { logInfo, logError } = require('./log')
+const { logInfo, logError } = require('../log')
 
 const Matchers = {
   host: host => ({ host: [host] }),
@@ -17,13 +17,13 @@ const Handlers = {
   })
 }
 
-class CaddyClient {
+class CaddyService {
 
-  static deploy(platformConfig, deploymentRequest) {
+  static addDeployment(platformConfig, deploymentRequest) {
 
     const url = `${deploymentRequest.subdomain}.${platformConfig.activeEnvironment}.${platformConfig.domain}`
 
-    logInfo({ message: `[CaddyClient] Updating caddy config to route ${url} to localhost:${deploymentRequest.hostPort}` })
+    logInfo({ message: `[CaddyService] Updating caddy config to route ${url} to localhost:${deploymentRequest.hostPort}` })
 
     return fetch(`http://localhost:2019/config/apps/http/servers/${platformConfig.ciServerName}/routes/0/handle`, {
       method: 'POST',
@@ -35,11 +35,11 @@ class CaddyClient {
         }
       ]))
     })
-      .catch(error => logError({ message: `[CaddyClient] Could not update caddy config to include ${url}`, error }))
+      .catch(error => logError({ message: `[CaddyService] Could not update caddy config to include ${url}`, error }))
   }
 
-  static applyDefaultServerConfig(platformConfig) {
-    logInfo({ message: '[CaddyClient] Applying default caddy config' })
+  static applyDefaultConfig(platformConfig) {
+    logInfo({ message: '[CaddyService] Applying default caddy config' })
     return fetch('http://localhost:2019/load', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
@@ -101,11 +101,11 @@ class CaddyClient {
         }
       })
     })
-      .catch(error => logError({ message: '[CaddyClient] Could not apply default caddy config', error }))
+      .catch(error => logError({ message: '[CaddyService] Could not apply default caddy config', error }))
   }
 
 }
 
 module.exports = {
-  CaddyClient
+  CaddyService
 }

package/src/server/cli.js

@@ -0,0 +1,34 @@
+#!/usr/bin/env node
+const arg = require('arg')
+const { exec } = require('child_process')
+const { PlatformClient } = require('./platform-server')
+
+const { logInfo, logError } = require('../log')
+
+//eslint-disable-next-line no-unused-vars
+const [_, __, command, ...restArgs] = process.argv
+
+const start = cliArgs => {
+  logInfo({ message: 'Running start command' })
+
+  const parsedArgs = arg({
+    '--platforms': String,
+    '-p': '--platforms'
+  }, { argv: cliArgs })
+
+  PlatformClient.start(parsedArgs['--platforms'])
+}
+
+const status = () => {
+  logInfo({ message: 'Running status command' })
+  exec('systemctl status deployment-tools.service')
+}
+
+const stop = () => {
+  logInfo({ message: 'Running stop command' })
+  exec('systemctl stop deployment-tools.service')
+}
+
+!!command
+  ? { start, status, stop }[command]()
+  : logError({ message: 'No command provided' })

package/src/{docker-client.js → server/docker.js}

@@ -1,12 +1,13 @@
 const { exec } = require('child_process')
 const fs = require('fs')
 const { nanoid } = require('nanoid')
-const { logInfo, logError } = require('./log')
 
-class DockerClient {
+const { logInfo, logError } = require('../log')
+
+class DockerService {
 
   static createDockerNetworkForContainerManagerServer(platformConfig) {
-    logInfo({ message: '[DockerClient] Creating docker network for comunication between containers' })
+    logInfo({ message: '[DockerService] Creating docker network for comunication between containers' })
     exec(`sudo docker network create ${platformConfig.ciDockerNetworkName}`)
   }
 
@@ -41,9 +42,9 @@ class DockerClient {
       logInfo({ message: 'Starting docker deployment sequence' })
 
       const dockerCommandScript = `'#!/bin/bash'
-${DockerClient.stopContainer(deploymentRequest)}
-${DockerClient.resolveCredentials(deploymentRequest)}
-${DockerClient.startContainer(platformConfig, deploymentRequest)}`
+${DockerService.stopContainer(deploymentRequest)}
+${DockerService.resolveCredentials(deploymentRequest)}
+${DockerService.startContainer(platformConfig, deploymentRequest)}`
 
       const deploymentId = nanoid()
 
@@ -55,15 +56,15 @@ ${DockerClient.startContainer(platformConfig, deploymentRequest)}`
       const command = exec(`bash ${FilePaths.DeploymentScript}`)
 
       command.stdout.on('data', data => {
-        logInfo({ message: `[DockerClient]: ${data}` })
+        logInfo({ message: `[DockerService]: ${data}` })
       })
 
       command.stderr.on('data', (data) => {
-        logError({ message: `[DockerClient]: ${data}` })
+        logError({ message: `[DockerService]: ${data}` })
       })
 
       command.on('close', code => {
-        logInfo({ message: `[DockerClient] command exited with code ${code}` })
+        logInfo({ message: `[DockerService] command exited with code ${code}` })
         fs.unlinkSync(FilePaths.DeploymentScript)
         resolve()
       })
@@ -74,5 +75,5 @@ ${DockerClient.startContainer(platformConfig, deploymentRequest)}`
 }
 
 module.exports = {
-  DockerClient
+  DockerService
 }

package/src/server/platform-server.js

@@ -0,0 +1,37 @@
+const { CaddyService } = require('./caddy')
+const { DockerService } = require('./docker')
+const { RestApiService } = require('./rest-api')
+
+const { PlatformTemplateService } = require('../template')
+const { PlatformConfigService } = require('../config')
+const { DeploymentQueueClient } = require('../deployment-queue')
+const { logError } = require('../log')
+
+// journalctl -u service-name.service
+class PlatformServerService {
+
+  static start(platformTemplatesFilePath) {
+    PlatformTemplateService.readFromFile(platformTemplatesFilePath).then(([firstPlatformTemplateFound]) => {
+      const platformConfig = PlatformConfigService.from(firstPlatformTemplateFound)
+
+      RestApiService.start(platformConfig)
+      CaddyService.applyDefaultConfig(platformConfig)
+
+      DeploymentQueueClient.pollForDeploymentRequests(
+        platformConfig,
+        deploymentRequest => {
+          DockerService.deploy(platformConfig, deploymentRequest)
+          CaddyService.addDeployment(platformConfig, deploymentRequest)
+          return Promise.resolve()
+        }
+      )
+
+    })
+      .catch(error => logError({ message: '[PlatformServerService] Could not start the platform server', error }))
+  }
+
+}
+
+module.exports = {
+  PlatformServerService
+}

package/src/{ci-rest-api.js → server/rest-api.js}

@@ -1,7 +1,7 @@
 const express = require('express')
-const { logInfo } = require('./log')
+const { logInfo } = require('../log')
 
-class CiRestApi {
+class RestApiService {
 
   static start(platformConfig) {
     const server = express()
@@ -17,12 +17,12 @@ class CiRestApi {
     })
 
     server.listen(platformConfig.ciInternalServerPort, () => {
-      logInfo({ message: `[ContainerManagerServer] API is live on port ${platformConfig.ciInternalServerPort}`})
+      logInfo({ message: `[RestApiService] API is live on port ${platformConfig.ciInternalServerPort}`})
     })
   }
 
 }
 
 module.exports = {
-  CiRestApi
+  RestApiService
 }

package/src/template/cli.js

@@ -0,0 +1,22 @@
+#!/usr/bin/env node
+const arg = require('arg')
+const { PlatformTemplateService } = require('./platform-template')
+
+const { logInfo, logError } = require('../log')
+
+//eslint-disable-next-line no-unused-vars
+const [_, __, command, ...options] = process.argv
+
+const validate = () => {
+  logInfo({ message: 'Running validate command' })
+
+  const parsedArgs = arg({
+    '--platforms': String,
+    '-p': '--platforms'
+  }, { argv: options })
+
+  PlatformTemplateService.readFromFile(parsedArgs['--platforms'])
+    .then(() => logInfo({ message: 'Template is valid' }))
+}
+
+command === 'validate' ? validate() : logError({ message: 'No command provided' })

package/src/template/index.js

@@ -0,0 +1 @@
+module.exports = require('./platform-template')

package/src/{platform-template.js → template/platform-template.js}

@@ -1,6 +1,6 @@
 const { resolve } = require('path')
 const { readFileSync } = require('fs')
-const { logError, logInfo } = require('./log')
+const { logError, logInfo } = require('../log')
 
 /**
  * Platform template definition
@@ -19,6 +19,7 @@ const { logError, logInfo } = require('./log')
  * @property {string} ciInternalServerPort- ? | number;
  * @property {string} ciServerName - ?
  * @property {string} ciDockerNetworkName - ?
+ * @property {string} ciGithubActionsRepo - organisation/repoName
  */
 class PlatformTemplateService {