@ossy/deployment-tools 0.0.85 → 0.0.89

package/README.md

@@ -42,3 +42,10 @@ The `cdk.json` file tells the CDK Toolkit how to execute your app.
 * `cdk deploy`      deploy this stack to your default AWS account/region
 * `cdk diff`        compare deployed stack with current state
 * `cdk synth`       emits the synthesized CloudFormation template
+
+## Useful commands
+
+```
+// journalctl -u deployment-tools.service
+// docker logs
+```

package/cdk.context.json

@@ -76,5 +76,13 @@
   "hosted-zone:account=858451553223:domainName=oskarssylwan.com:region=eu-north-1": {
     "Id": "/hostedzone/Z07488011P8FXGWA11LTS",
     "Name": "oskarssylwan.com."
+  },
+  "hosted-zone:account=858451553223:domainName=plexus-sanitas.com:region=eu-north-1": {
+    "Id": "/hostedzone/Z0222639MQIJ8EOL9VAG",
+    "Name": "plexus-sanitas.com."
+  },
+  "hosted-zone:account=858451553223:domainName=tepit.se:region=eu-north-1": {
+    "Id": "/hostedzone/Z07859112SU1GH5SKMUXD",
+    "Name": "tepit.se."
   }
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ossy/deployment-tools",
-  "version": "0.0.85",
+  "version": "0.0.89",
   "description": "Collection of scripts and tools to aid deployment of containers and static files to Amazon Web Services through GitHub Actions",
   "source": "./src/index.js",
   "main": "./src/index.js",
@@ -22,6 +22,7 @@
     "aws-cdk-lib": "^2.73.0",
     "constructs": "^10.1.304",
     "express": "^4.18.1",
+    "get-port": "^7.1.0",
     "glob": "^9.3.2",
     "nanoid": "^3.3.4",
     "node-fetch": "^2.6.7"

package/src/aws-credentials/cli.js

@@ -1,6 +1,5 @@
 const arg = require('arg')
 const { AwsCredentialsService } = require('./aws-credentials')
-
 const { PlatformTemplateService } = require('../template')
 const { PlatformConfigService } = require('../config')
 const { logInfo, logError } = require('../log')

package/src/caddy/caddy-config.js

@@ -1,5 +1,3 @@
-const { DeploymentTemplateService } = require('../template')
-
 const Matchers = {
   host: host => ({ host: [host] }),
   path: path => ({ path: [path] })
@@ -21,60 +19,66 @@ const Handlers = {
 */
 class CaddyConfigService {
 
-  static createConfig(platformConfig, deploymentTemplates) {
-
-    const containerDeployments = DeploymentTemplateService
-      .getContainerDeployments(deploymentTemplates)
-
+  static createServer() {
     return {
       apps: {
+        tls: CaddyConfigService.tlsIssuers(),
         http: {
           servers: {
             'deployment-tools': {
-              listen: [':80', ':443'],
-              routes: containerDeployments.map(deploymentTemplate => ({
-                match: [Matchers.host(deploymentTemplate.domain)],
-                handle: [Handlers.subroute([{ handle: [Handlers.reverseProxy(deploymentTemplate.hostPort)]}])]
-              }))
+              listen: [':443'],
+              routes: []
             }
           }
         },
-        tls: {
-          automation: {
-            policies: [
-              {
-                subjects: containerDeployments.map(({ domain }) => domain),
-                issuers:[
-                  {
-                    challenges: {
-                      dns:{
-                        provider: {
-                          'max_retries': 10,
-                          name: 'route53',
-                          'aws_profile': 'ci-client'
-                        }
+      }
+    }
+  }
+
+  static createProxyConfig(containerDeploymentTemplate) {
+    return {
+      '@id': containerDeploymentTemplate.domain,
+      match: [Matchers.host(containerDeploymentTemplate.domain)],
+      handle: [Handlers.subroute([{ handle: [Handlers.reverseProxy(containerDeploymentTemplate.hostPort)]}])]
+    }
+  }
+
+  static tlsIssuers() {
+    // add tls automation to the server config under apps.tls property
+    return {
+        automation: {
+          policies: [
+            {
+              subjects: [],
+              issuers:[
+                {
+                  challenges: {
+                    dns:{
+                      provider: {
+                        'max_retries': 10,
+                        name: 'route53',
+                        'aws_profile': 'ci-client'
                       }
-                    },
-                    module: 'acme'
+                    }
                   },
-                  {
-                    challenges:{
-                      dns:{
-                        provider:{
-                          'max_retries': 10,
-                          name: 'route53'
-                        }
+                  module: 'acme'
+                },
+                {
+                  challenges:{
+                    dns:{
+                      provider:{
+                        'max_retries': 10,
+                        name: 'route53'
                       }
-                    },
-                    module: 'zerossl'
-                  }
-                ]
-              }
-            ]
-          }
+                    }
+                  },
+                  module: 'zerossl'
+                }
+              ]
+            }
+          ]
         }
       }
-    }
   }
 
 }

package/src/caddy/caddy.js

@@ -1,13 +1,14 @@
 const fetch = require('node-fetch')
+const { logError, logInfo } = require('../log')
 const { CaddyConfigService } = require('./caddy-config')
-const { logInfo, logError, logDebug } = require('../log')
 
 /**
   * @class
 */
 class CaddyService {
 
-  static applyConfig(config) {
+  static loadServerConfig() {
+    const config = CaddyConfigService.createServer()
     return fetch('http://localhost:2019/load', {
       method: 'POST',
       headers: { 'Content-Type': 'application/json' },
@@ -17,6 +18,90 @@ class CaddyService {
     .catch(error => logError({ message: '[CaddyService] Could not apply default caddy config', error }))
   }
 
+  static apply(containerDeploymentTemplate) {
+    return CaddyService.applyProxyConfig(containerDeploymentTemplate)
+      .then(() => CaddyService.applyTlsSubject(containerDeploymentTemplate))
+  }
+
+  static getConfig() {
+    return fetch('http://localhost:2019/config/', {
+      method: 'GET',
+      headers: { 'Content-Type': 'application/json' }
+    }).then(response => response.json())
+  }
+
+  static getTlsSubjects() {
+    return fetch('http://localhost:2019/config/apps/tls/automation/policies/0/subjects', {
+      method: 'GET',
+      headers: { 'Content-Type': 'application/json' }
+    })
+    .then(response => response.json())
+    .catch(error => logError({ message: '[CaddyService] Could not get tls subjects', error }))
+  }
+
+  static applyTlsSubject(containerDeploymentTemplate) {
+    return CaddyService.getTlsSubjects()
+      .then(tlsSubjects => {
+        const subjectExists = tlsSubjects.includes(containerDeploymentTemplate.domain)
+        return subjectExists
+          ? Promise.resolve()
+          : CaddyService.addTlsSubject(containerDeploymentTemplate)
+      })
+  }
+
+  static addTlsSubject(containerDeploymentTemplate) {
+    return fetch('http://localhost:2019/config/apps/tls/automation/policies/0/subjects', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(containerDeploymentTemplate.domain)
+    })
+    .then(response => response?.error && logError({ message: '[CaddyService] Could not add tls subject', error }))
+    .catch(error => logError({ message: '[CaddyService] Could not add tls subject', error }))
+  }
+
+  static getProxyConfig(containerDeploymentTemplate) {
+    return fetch(`http://localhost:2019/id/${containerDeploymentTemplate.domain}`, {
+      method: 'GET',
+      headers: { 'Content-Type': 'application/json' }
+    })
+    .then(response => response.json())
+    .then(response => (!response || response?.error) ? Promise.reject() : response)
+    .catch(error => {
+      logInfo({ message: `[CaddyService] Could not find a proxy for ${containerDeploymentTemplate.domain}`, error })
+      return undefined
+    })
+  }
+
+  static applyProxyConfig(containerDeploymentTemplate) {
+    return CaddyService.getProxyConfig(containerDeploymentTemplate)
+      .then(proxyConfig => {
+          proxyConfig
+              ? CaddyService.replaceProxyConfig(containerDeploymentTemplate)
+              : CaddyService.addProxyConfig(containerDeploymentTemplate)
+      })
+  }
+
+  static addProxyConfig(containerDeploymentTemplate) {
+    const proxyConfig = CaddyConfigService.createProxyConfig(containerDeploymentTemplate)
+    return fetch('http://localhost:2019/config/apps/http/servers/deployment-tools/routes', {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(proxyConfig)
+    })
+    .then(response => response?.error && logError({ message: '[CaddyService] Could not add proxy config', error }))
+    .catch(error => logError({ message: '[CaddyService] Could not add proxy config', error }))
+  }
+
+  static replaceProxyConfig(containerDeploymentTemplate) {
+    const proxyConfig = CaddyConfigService.createProxyConfig(containerDeploymentTemplate)
+    return fetch(`http://localhost:2019/id/${containerDeploymentTemplate.domain}`, {
+      method: 'POST',
+      headers: { 'Content-Type': 'application/json' },
+      body: JSON.stringify(proxyConfig)
+    })
+    .then(response => response?.error && logError({ message: '[CaddyService] Could not add proxy config', error }))
+    .catch(error => logError({ message: '[CaddyService] Could not add proxy config', error }))
+  }
 
 }
 

package/src/caddy/caddy.playground.js

@@ -0,0 +1,31 @@
+const { CaddyService } = require('./caddy')
+
+const containerDeploymentTemplate = {
+    "domain": "peter.ossy.se",
+    "image": "ossy-se/website-alex",
+    "targetDeploymentPlatform": "ossybot",
+    "type": "CONTAINER",
+    "hostPort": "3016"
+}
+
+
+// CaddyService.getProxyForDomain('alex.ossy.se')
+//  .then(x => console.log(JSON.stringify(x, null, 2)))
+
+//  CaddyService.getProxyForDomain('ossy.se')
+//   .then(x => console.log(JSON.stringify(x, null, 2)))
+
+// OnStartup
+// CaddyService.loadServerConfig()
+
+// Get config
+CaddyService.getConfig()
+    .then(x => console.log(JSON.stringify(x, null, 2)))
+//     .then(() => CaddyService.applyProxyConfig(containerDeploymentTemplate))
+//     .then(() => CaddyService.getConfig())
+//     .then(x => console.log(JSON.stringify(x, null, 2)))
+
+// CaddyService.getTlsSubjects()
+    // .then(x => console.log(JSON.stringify(x, null, 2)))
+
+// CaddyService.apply(containerDeploymentTemplate)

package/src/deploy/platform-deployment.js

@@ -1,9 +1,6 @@
-const { readFileSync } = require('fs')
-const { resolve } = require('path')
 const { PlatformTemplateService, DeploymentTemplateService } = require('../template')
 const { PlatformConfigService, SupportedDeploymentTypes } = require('../config')
 const { DeploymentQueueService } = require('../deployment-queue')
-const { CaddyConfigService } = require('../caddy')
 const { logError } = require('../log')
 
 /**
@@ -45,13 +42,10 @@ class PlatformDeploymentService {
 
         if (deploymentTemplate.type === SupportedDeploymentTypes.Container) {
 
-          const caddyConfig = CaddyConfigService.createConfig(platformConfig, deploymentTemplatesForTargetPlatform)
-
           const deploymentRequest = {
             ...deploymentTemplate,
             username: username,
             authentication: authentication,
-            caddyConfig
           }
 
           return DeploymentQueueService.sendDeploymentRequest(platformConfig, deploymentRequest)
@@ -97,13 +91,10 @@ class PlatformDeploymentService {
         return deploymentTemplatesForTargetPlatform.map(deploymentTemplate => {
           if (deploymentTemplate.type === SupportedDeploymentTypes.Container) {
 
-            const caddyConfig = CaddyConfigService.createConfig(platformConfig, deploymentTemplatesForTargetPlatform)
-
             const deploymentRequest = {
               ...deploymentTemplate,
               username: username,
               authentication: authentication,
-              caddyConfig
             }
 
             return DeploymentQueueService.sendDeploymentRequest(platformConfig, deploymentRequest)

package/src/docker/docker-service.js

@@ -27,7 +27,7 @@ class DockerService {
 
   static stopContainer(deploymentRequest) {
     logInfo({ message: `[DockerService] Stopping container for ${deploymentRequest.domain}` })
-    return exec(`docker stop ${deploymentRequest.domain}`)
+    return exec(`docker stop c-${deploymentRequest.domain}`)
       .catch(() => {}) // no worries if container isn't there
   }
 
@@ -46,12 +46,18 @@ class DockerService {
       .catch(logErrorAndReject(`[DockerService] Could pull image ${image}:latest`))
   }
 
-  static startContainer(platformConfig, { image, containerPort, hostPort, registry, env, domain }) {
+  static startContainer(platformConfig, { image, containerPort, registry, env, domain }) {
     const imageUrl = !!registry ? `${registry}/${image}` : image
     const envsAsString = Object.entries(env || {}).reduce((envs, [name, value]) => `${envs} --env ${name}="${value}"`, '')
-    logInfo({ message: `[DockerService] Starting container for ${domain} with port mapping ${hostPort}:${containerPort} and source ${imageUrl}` })
-    return exec(`docker run -d -p ${hostPort}:${containerPort} --name=${domain} --network=${platformConfig.ciDockerNetworkName} --network-alias=${domain} --rm ${envsAsString} ${imageUrl}`)
-      .catch(logErrorAndReject(`[DockerService] Could not start container ${image}`))
+    
+    return DockerService.getUnusedPort()
+      .then(hostPort => {
+        logInfo({ message: `[DockerService] Starting container for ${domain} with port mapping ${hostPort}:${containerPort} and source ${imageUrl}` })
+        return exec(`docker run -d -p ${hostPort}:${containerPort} --name=c-${domain} --network=${platformConfig.ciDockerNetworkName} --network-alias=c-${domain} --rm ${envsAsString} ${imageUrl}`)
+          .then(() => ({ hostPort }))
+          .catch(logErrorAndReject(`[DockerService] Could not start container ${image}`))
+      })
+      
   }
 
   static resolveCredentials({ registry, username, authentication }) {
@@ -78,6 +84,17 @@ class DockerService {
       .then(() => DockerService.startContainer(platformConfig, deploymentRequest))
   }
 
+  static getUnusedPort() {
+    // get-port is written in es6, so we need to import it like this
+    return import('get-port')
+      .then(module => {
+        const getPort = module.default;
+        const portNumbers = module.portNumbers;
+    
+        return getPort({ port: portNumbers(3000, 3200) })
+      });
+  }
+
 }
 
 module.exports = {

package/src/docker/docker-service.playground.js

@@ -1,8 +1,5 @@
 const { DockerService } = require('./docker-service')
 
-const platformConfig = {
-  ciDockerNetworkName: 'foo'
-}
 
 const deploymentTemplate =   {
     "domain": "api.qa.ossy.se",

package/src/index.cli.js

@@ -8,7 +8,6 @@ const loadHandler = {
   deployment: () => require('./deploy/cli.js'),
   template: () => require('./template/cli.js'),
   server: () => require('./server/cli.js'),
-  cms: () => require('./cms/cli.js')
 }[handlerName]
 
 !!loadHandler && loadHandler().handler(restArgs)

package/src/infrastructure/cli.js

@@ -1,12 +1,11 @@
 #!/usr/bin/env node
 /* eslint-disable no-new */
-const { App, CfnOutput } = require('aws-cdk-lib')
+const { App } = require('aws-cdk-lib')
 const { TrustCiStack } = require('./trust-ci-stack')
 const { DeploymentTargetStack } = require('./deployment-target-stack')
 const { DnsStack } = require('./dns-stack')
 const { PlatformTemplateService, DeploymentTemplateService } = require('../template')
 const { PlatformConfigService } = require('../config')
-const { PlatformDeploymentService } = require('../deploy')
 
 Promise.all([
   DeploymentTemplateService.readFromFiles(process.env.DEPLOYMENTS),

package/src/infrastructure/deployment-target-stack.js

@@ -3,6 +3,8 @@ const { nanoid } = require('nanoid')
 const { CfnOutput, Stack, Duration, RemovalPolicy } = require('aws-cdk-lib')
 const { HostedZone, ARecord, RecordTarget } = require('aws-cdk-lib/aws-route53')
 const { Bucket, BucketEncryption, BlockPublicAccess } = require('aws-cdk-lib/aws-s3')
+const { Distribution, PriceClass } = require('aws-cdk-lib/aws-cloudfront')
+const { S3Origin } = require('aws-cdk-lib/aws-cloudfront-origins')
 const { ContainerDeploymentTarget } = require('./container-deployment-target')
 
 /**
@@ -16,17 +18,33 @@ class DeploymentTargetStack extends Stack {
       throw ('[DeploymentTargetStack] No template provided')
     }
 
-    const bucketId = nanoid().toLowerCase().replaceAll('_', '').replaceAll('-', '')
-    const bucketName = `${props.config.platformName}-${bucketId}`
+    // create a bucket name before running cdk deploy and put it in config
+    // so that it won't change when updating resources
+    // const bucketId = nanoid().toLowerCase().replaceAll('_', '').replaceAll('-', '')
+    // const bucketName = `${props.config.platformName}-${bucketId}`
 
-    // TODO: this should probably not be destroyed....
+    // TODO: Check if the bucket already exists and use it
+    // instead of having the whole deployment fail
     const staticDeploymentTarget =
       new Bucket(this, 'StaticDeploymentTarget', {
-        bucketName: bucketName,
+        bucketName: props.config.awsStaticBucketName,
         blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
-        encryption: BucketEncryption.S3_MANAGED,
-        removalPolicy: RemovalPolicy.DESTROY,
-        autoDeleteObjects: true
+        encryption: BucketEncryption.UNENCRYPTED,
+        removalPolicy: RemovalPolicy.RETAIN,
+        autoDeleteObjects: false,
+        cors: [{
+          allowedHeaders: ['*'],
+          allowedMethods: ['GET', 'PUT', 'POST' ],
+          allowedOrigins: [ '*' ],
+          exposeHeaders: []
+        }]
+      })
+
+    const mediaCDN = new Distribution(this, 'Media', {
+        defaultBehavior: {
+          origin: new S3Origin(staticDeploymentTarget, { originPath: '/media' }),
+        },
+        priceClass: PriceClass.PRICE_CLASS_100
       })
 
     const containerDeploymentTarget =
@@ -39,6 +57,12 @@ class DeploymentTargetStack extends Stack {
     this.staticDeploymentTargetBucketName = staticDeploymentTarget.bucketName
     this.staticDeploymentTargetBucketArn = staticDeploymentTarget.bucketArn
 
+    new CfnOutput(this, 'MediaCDNDomainName', {
+      value: mediaCDN.domainName,
+      description: 'Domain name of CDN distribution',
+      exportName: 'MediaCDNDomainName'
+    })
+
     new CfnOutput(this, 'ContainerDeploymentTargetPublicIp', {
       value: containerDeploymentTarget.instancePublicIp,
       description: 'Public ip of the ec2 instance',

package/src/server/platform-server.js

@@ -1,33 +1,35 @@
 const { CaddyService } = require('../caddy')
 const { DockerService } = require('../docker')
-
 const { PlatformTemplateService } = require('../template')
 const { PlatformConfigService } = require('../config')
 const { DeploymentQueueService } = require('../deployment-queue')
 const { logError } = require('../log')
 
-// journalctl -u deployment-tools.service
-// docker logs
 /**
   * @class
 */
 class PlatformServerService {
 
   static start(platformTemplatesFilePath) {
-    PlatformTemplateService.readFromFile(platformTemplatesFilePath).then(([firstPlatformTemplateFound]) => {
-      const platformConfig = PlatformConfigService.from(firstPlatformTemplateFound)
-
-      DockerService.createDockerNetworkForContainerManagerServer(platformConfig)
-        .then(() => DockerService.startDefaultContainers(platformConfig))
-
-      DeploymentQueueService.pollForDeploymentRequests(
-        platformConfig,
-        deploymentRequest => {
-          DockerService.deploy(platformConfig, deploymentRequest)
-            .then(() => CaddyService.applyConfig(deploymentRequest.caddyConfig))
-          return Promise.resolve()
-        }
-      )
+    PlatformTemplateService
+      .readFromFile(platformTemplatesFilePath)
+      .then(([firstPlatformTemplateFound]) => {
+        const platformConfig = PlatformConfigService.from(firstPlatformTemplateFound)
+
+        DockerService.createDockerNetworkForContainerManagerServer(platformConfig)
+          .then(() => DockerService.startDefaultContainers(platformConfig))
+
+        CaddyService.loadServerConfig()
+
+        DeploymentQueueService.pollForDeploymentRequests(
+          platformConfig,
+          deploymentRequest => {
+            DockerService.deploy(platformConfig, deploymentRequest)
+              .then(({ hostPort }) => CaddyService.apply({ ...deploymentRequest, hostPort }))
+
+            return Promise.resolve()
+          }
+        )
 
     })
       .catch(error => logError({ message: '[PlatformServerService] Could not start the platform server', error }))

package/src/template/deployment-template.js

@@ -1,6 +1,5 @@
 const { glob } = require('glob')
 const { readFileSync } = require('fs')
-const { logError, logInfo } = require('../log')
 const { SupportedDeploymentTypes } = require('../config')
 
 /**

package/src/cms/cli.js

@@ -1,55 +0,0 @@
-#!/usr/bin/env node
-const { resolve } = require('path')
-const { readFileSync } = require('fs')
-const arg = require('arg')
-const fetch = require('node-fetch')
-const { logInfo, logError } = require('../log')
-
-const config = {
-  apiUrl: 'https://api.ossy.se/api/v0'
-}
-
-const getTokenPayload = token =>
-  JSON.parse(Buffer.from(token.split('.')[1], 'base64').toString())
-
-const importResourceTemplates = options => {
-
-  const parsedArgs = arg({
-    '--authentication': String,
-    '--a': '--authentication',
-
-    '--ossy-file': String,
-    '-o': '--ossy-file',
-  }, { argv: options })
-
-  logInfo({ message: '[CMS] reading files' })
-  const token = parsedArgs['--authentication']
-  const tokenPayload = getTokenPayload(token)
-  const ossyfile = JSON.parse(readFileSync(resolve(parsedArgs['--ossy-file']), 'utf8'))
-
-  if (!token) return logError({ message: '[CMS] No token provided with --authentication'})
-  if (!ossyfile?.workspaceId) return logError({ message: '[CMS] No workspaceId provided in ossy.json'})
-  if (!ossyfile?.resourceTemplates) return logError({ message: '[CMS] No resource templates provided in ossy.json'})
-
-  logInfo({ message: '[CMS] uploading resource templates' })
-
-  const endpoint = `${config.apiUrl}/workspaces/${ossyfile.workspaceId}/resource-templates`
-
-  const fetchOptions = {
-    method: 'POST',
-    headers: { 'Authorization': token, 'Content-Type': 'application/json' },
-    body: JSON.stringify(ossyfile.resourceTemplates)
-  }
-
-  fetch(endpoint, fetchOptions)
-    .then(() => logInfo({ message: '[CMS] Done' }))
-    .catch(error => logError({ message: '[CMS] Error', error }))
-}
-
-module.exports = {
-  handler: ([command, ...options]) => {
-    !!command
-      ? { 'import-resource-templates': importResourceTemplates }[command](options)
-      : logError({ message: '[CMS] No command provided' })
-  }
-}

package/src/ossy.json

@@ -1,14 +0,0 @@
-{
-  "workspaceId": "-LWn40rqowQXuNhm4BTvz",
-  "resourceTemplates": [
-    {
-      "name": "Consulting profile",
-      "fields": [
-        {
-          "name": "Name",
-          "type": "string"
-        }
-      ]
-    }
-  ]
-}