@ossy/deployment-tools 0.0.67 → 0.0.68

package/README.md

@@ -37,11 +37,55 @@ npx --yes @ossy/deployment-tools deploy \
   --ossyfile packages/${{ github.event.inputs.packageName }}/ossy.json \
 ```
 
-## Infrastructure
+## Concepts
+
+**Workspace**
+
+A workspace is an umbrella for our services.
+It needs to be associated with at least one billable account.
+The workspace holds information like what tools and services are in use and should
+be billed, and what users have access to these tools and services.
+
+```
+id: string;
+name: string;
+participants: WorkspaceParticipant[];
+services: ServiceDefinition[];
+billingInformation
+
+```
+
+## Overview of our infrastructure
 
 We use AWS to host our infrastructure and all of it is defined in JavaScript with the help of
 (AWS CDK)[https://aws.amazon.com/cdk/].
 
+**Static content**
+
+We use a s3 bucket for static content.
+This bucket is used to host websites, images, videos and other static content.
+On root level you'll find directories that represent one workspace each.
+
+/<workspacId>/<service>/
+/<workspacId>/websites/website-id
+
+The bucket have directories for each workspace that is the workspaceID.
+
+The bucket have directories for each workspace that contains a media directory and website directories
+
+- a place to host media files like images, videos, pdf documents etc.
+- a platform to host different docker images to
+- a mongodb database that can ensure data persistance without much effort from our our side
+- an email service
+- an easy way to host multiple single page applications
+
+Stacks
+
+- email service stack per env
+- media bucket stack per env
+- stack for dns records with the account that holds domain names
+-
+
 ### Adding a new AWS account
 We have a different account for each service and environment.
 To add a new account follow the steps below.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@ossy/deployment-tools",
-  "version": "0.0.67",
+  "version": "0.0.68",
   "description": "Collection of scripts and tools to aid deployment of containers and static files to Amazon Web Services through GitHub Actions",
   "source": "./src/index.js",
   "main": "./src/index.js",

package/src/caddy/caddy-config.js

@@ -23,7 +23,7 @@ class CaddyConfigService {
 
     const url = [
       deploymentRequest.subdomain,
-      platformConfig.activeEnvironment !== 'prod' ? platformConfig.activeEnvironment : undefined,
+      platformConfig.environmentType !== 'prod' ? platformConfig.environmentType : undefined,
       platformConfig.domain
     ]
       .filter(x => !!x)
@@ -44,18 +44,18 @@ class CaddyConfigService {
               listen: [':80', ':443'],
               routes: [
                 {
-                  match: [Matchers.host(`${platformConfig.ciSubDomain}.${platformConfig.activeEnvironment}.${platformConfig.domain}`)],
+                  match: [Matchers.host(`${platformConfig.ciSubDomain}.${platformConfig.environmentType}.${platformConfig.domain}`)],
                   handle: [Handlers.subroute([{ handle: [Handlers.reverseProxy(platformConfig.ciInternalServerPort)]}])]
                 }
               ]
             }
           }
         },
-        tls: platformConfig.activeEnvironment === 'local-dev' ? undefined : {
+        tls: platformConfig.environmentType === 'local-dev' ? undefined : {
           automation: {
             policies: [
               {
-                subjects:[`*.${platformConfig.activeEnvironment}.${platformConfig.domain}`],
+                subjects:[`*.${platformConfig.environmentType}.${platformConfig.domain}`],
                 issuers:[
                   {
                     challenges:{

package/src/caddy/caddy.playground.js

@@ -1,7 +1,7 @@
 const { CaddyService } = require('./caddy')
 
 const config = {
-  activeEnvironment: 'prod',
+  environmentType: 'prod',
   domain: 'localhost',
   ciServerName: 'ossy',
   ciSubDomain: 'ci',

package/src/config/platform-config.js

@@ -1,3 +1,23 @@
+/**
+ * Platform config definition
+ * @typedef {Object} PlatformConfig
+ * @property {string} platformName - Name of platform
+ * @property {string} domain - example.com
+ * @property {string} environmentType - local, test, qa, prod
+ *
+ * @property {string} awsAccountId - Aws account id
+ * @property {string=} awsRegion - ?
+ * @property {string=} awsKeyPairName - ?
+ * @property {string} awsRoleToAssume - ?
+ * @property {string=} awsDeploymentSqsArn - ?
+ *
+ * @property {string=} ciSubDomain - ?
+ * @property {string|mumber=} ciInternalServerPort - ? | number;
+ * @property {string=} ciServerName - ?
+ * @property {string=} ciDockerNetworkName - ?
+ * @property {string} ciGithubActionsRepo - organisation/repoName
+ */
+
 const SupportedRegions = {
   North: 'eu-north-1'
 }
@@ -11,16 +31,10 @@ const SupportedEnvironments = {
 }
 
 const SupportedDeploymentTypes = {
-  Container: 'CONTAINER'
-  // Static = 'STATIC'
+  Container: 'CONTAINER',
+  Static: 'STATIC'
 }
 
-// export interface PlatformConfig extends Required<Omit<PlatformTemplate, 'awsRoleToAssume' | 'awsKeyPairName'>> {
-//   activeEnvironment: SupportedEnvironments;
-//   awsRoleToAssume?: string;
-//   awsKeyPairName?: string;
-// }
-
 /**
   * @class
 */
@@ -31,17 +45,18 @@ class PlatformConfigService {
     const withDefaults = {
       platformName: SupportedEnvironments.LOCAL,
       domain: 'localhost',
-      activeEnvironment: SupportedEnvironments.LOCAL,
-      supportedDeploymentTypes: ['CONTAINER'],
+      environmentType: SupportedEnvironments.LOCAL,
+      awsRegion: SupportedRegions.North,
+      ...template,
+      // values below should not be overriden by the template properties
       ciSubDomain: 'ci',
       ciInternalServerPort: 3000,
       ciServerName: 'ci-client',
       ciDockerNetworkName: 'deployment-tools',
-      awsRegion: SupportedRegions.North,
-      ...template
     }
 
-    const awsDeploymentSqsArn = `https://sqs.${withDefaults.awsRegion}.amazonaws.com/${withDefaults.awsAccountId}/${withDefaults.platformName}-${withDefaults.activeEnvironment}`
+    const awsDeploymentSqsArn =
+      `https://sqs.${withDefaults.awsRegion}.amazonaws.com/${withDefaults.awsAccountId}/${withDefaults.platformName}-${withDefaults.environmentType}`
 
     const awsRoleToAssume = process.env.CI
       ? `github-ci-role-${withDefaults.platformName}`

package/src/deploy/platform-deployment.js

@@ -45,38 +45,38 @@ class PlatformDeploymentService {
     pathToOssyFile
   }) {
 
-    const platformConfigRequest = PlatformTemplateService.readFromFile(pathToPlatformTemplates)
-      .then(templates => templates.map(x => ({ ...x, activeEnvironment: targetEnvironment })))
-      .then(templates => templates.map(x => PlatformConfigService.from(x)))
-
-    const deploymentTemplatesRequest = PlatformDeploymentService.getDeploymentTemplates(pathToOssyFile)
-
-    return Promise.all([platformConfigRequest, deploymentTemplatesRequest])
-      .then(([platformConfigs, deploymentTemplates]) => {
+    return Promise.all([
+      PlatformDeploymentService.getDeploymentTemplates(pathToOssyFile),
+      PlatformTemplateService.readFromFile(pathToPlatformTemplates)
+        .then(templates => templates.map(PlatformConfigService.from))
+    ])
+      .then(([deploymentTemplates, platformConfigs]) => {
         deploymentTemplates.map(deploymentTemplate => {
 
-          const platformConfig = platformConfigs.find(config => config.platformName === deploymentTemplate.targetDeploymentPlatform)
+          const platformConfig = platformConfigs.find(config =>
+            config.platformName === deploymentTemplate.targetDeploymentPlatform
+            && config.environmentType == targetEnvironment
+          )
 
           if (!platformConfig) {
-            logError({ message: `[PlatformDeploymentService] Could not find a deployment platform with the name ${deploymentTemplate.targetDeploymentPlatform}` })
+            logError({ message: `[PlatformDeploymentService] Could not find a deployment platform with the name ${deploymentTemplate.targetDeploymentPlatform} and environment type ${targetEnvironment}` })
             return Promise.reject()
           }
 
           process.env.NODE_TLS_REJECT_UNAUTHORIZED = '0'
 
-          if (deploymentTemplate.type !== SupportedDeploymentTypes.Container) {
-            logError({ message: `[PlatformDeploymentService] Unsupported deployment type of ${deploymentTemplate.type}` })
-            return Promise.reject()
-          }
+          if (deploymentTemplate.type === SupportedDeploymentTypes.Container) {
+            const deploymentRequest = {
+              ...deploymentTemplate,
+              username: username,
+              authentication: authentication
+            }
 
-          const deploymentRequest = {
-            ...deploymentTemplate,
-            env: PlatformDeploymentService.getEnvironmentVariables(targetEnvironment, deploymentTemplate),
-            username: username,
-            authentication: authentication
+            return DeploymentQueueService.sendDeploymentRequest(platformConfig, deploymentRequest)
           }
 
-          return DeploymentQueueService.sendDeploymentRequest(platformConfig, deploymentRequest)
+          logError({ message: `[PlatformDeploymentService] Unsupported deployment type of ${deploymentTemplate.type}` })
+          return Promise.reject()
 
         })
       })
@@ -89,14 +89,6 @@ class PlatformDeploymentService {
     return Promise.resolve(ossyfile.deployments || [])
   }
 
-  static getEnvironmentVariables(targetEnvironment, deploymentRequest) {
-    const envs = deploymentRequest.env || {}
-    return {
-      ...(envs.shared || {}),
-      ...(envs[targetEnvironment] || {})
-    }
-  }
-
 }
 
 module.exports = {

package/src/infrastructure/cli.js

@@ -1,10 +1,8 @@
 #!/usr/bin/env node
 /* eslint-disable no-new */
 const { App } = require('aws-cdk-lib')
-
-const { PlatformStack } = require('./platform-stack')
-const { EstablishTrustStack } = require('./establish-trust-stack')
-
+const { TrustCiStack } = require('./trust-ci-stack')
+const { DeploymentTargetStack } = require('./deployment-target-stack')
 const { PlatformTemplateService } = require('../template')
 const { PlatformConfigService } = require('../config')
 
@@ -12,19 +10,21 @@ PlatformTemplateService
   .readFromFile(process.env.PLATFORMS)
   .then(templates => templates.map(PlatformConfigService.from))
   .then(configs => {
-
     const app = new App()
 
     configs.forEach(config => {
-      const env = { account: config.awsAccountId, region: config.awsRegion }
-
-      new EstablishTrustStack(app, `${config.platformName}-establish-trust`, { config, env })
-
-      config.supportedEnvironments
-        .map(activeEnvironment => PlatformConfigService.from({ ...config, activeEnvironment }))
-        .forEach(config => {
-          new PlatformStack(app, `${config.platformName}-${config.activeEnvironment}`, { config, env })
-        })
+      const stackBaseName = `${config.platformName}-${config.environmentType}`
+
+      const stackProps = {
+        config,
+        env: {
+          account: config.awsAccountId,
+          region: config.awsRegion
+        }
+      }
+
+      new TrustCiStack(app, `${stackBaseName}-trust-ci`, stackProps)
+      new DeploymentTargetStack(app, `${stackBaseName}-deployment-target`, stackProps)
 
     })
   })

package/src/infrastructure/container-server/container-server.js

@@ -1,4 +1,4 @@
-const { CfnOutput } = require('aws-cdk-lib')
+const { CfnOutput, Duration, RemovalPolicy } = require('aws-cdk-lib')
 const { Construct } = require('constructs')
 const {
   Instance,
@@ -12,8 +12,11 @@ const {
   Port,
   UserData
 } = require('aws-cdk-lib/aws-ec2')
-const { HostedZone, ARecord, RecordTarget } = require('aws-cdk-lib/aws-route53')
+const { ARecord, RecordTarget } = require('aws-cdk-lib/aws-route53')
 const { Role, ServicePrincipal, Policy, PolicyStatement, Effect } = require('aws-cdk-lib/aws-iam')
+const { Queue } = require('aws-cdk-lib/aws-sqs')
+const { Source, BucketDeployment } = require('aws-cdk-lib/aws-s3-deployment')
+
 const {
   getInstallNodeJs,
   getInstallNpm,
@@ -22,15 +25,18 @@ const {
 const { CaddyService } = require('./caddy.service')
 const { DeploymentToolsService } = require('./deployment-tools.service')
 const { AwsProfile } = require('./aws-profile')
+
 const { SupportedRegions } = require('../../config')
 
 /**
- * Platform template definition
+ * ContainerServerProps
  * @namespace ContainerServer
  * @typedef {Object} ContainerServerProps
- * @property {PlatformConfig} platformConfig - platform config
- * @property {Bucket} platformConfigBucket - aws bucket
+ * @property {PlatformConfig} config - platform config
+ * @property {Zone} hostedZone - aws zone
+ * @property {Bucket} bucket - s3 bucket
  */
+
 const InstanceImages = {
   UBUNTU: 'ami-092cce4a19b438926'
 }
@@ -41,8 +47,6 @@ const InstanceImages = {
 class ContainerServer extends Construct {
 
   /**
-   * EC2 Instance that docker containers are served on
-   *
    * @param {object} scope - scope
    * @param {string} id - id
    * @param {ContainerServerProps} props - ContainerServerProps
@@ -50,18 +54,8 @@ class ContainerServer extends Construct {
   constructor(scope, id, props) {
     super(scope, id)
 
-    const hostedZone = new HostedZone(this, 'HostedZone', {
-      zoneName: `${props.platformConfig.activeEnvironment}.${props.platformConfig.domain}`
-    })
-
-    const vpc = Vpc.fromLookup(this, 'VPC', {
-      isDefault: true
-    })
-
-    const securityGroup = new SecurityGroup(this, 'SecurityGroup', {
-      vpc,
-      allowAllOutbound: true
-    })
+    const vpc = Vpc.fromLookup(this, 'VPC', { isDefault: true })
+    const securityGroup = new SecurityGroup(this, 'SecurityGroup', { vpc, allowAllOutbound: true })
 
     securityGroup.addIngressRule(
       Peer.anyIpv4(),
@@ -81,6 +75,16 @@ class ContainerServer extends Construct {
       'allow HTTPS traffic from anywhere'
     )
 
+    const deploymentQueue = new Queue(this, 'DeploymentQueue', {
+      queueName: `${props.config.platformName}-${props.config.environmentType}`,
+      receiveMessageWaitTime: Duration.seconds(20)
+    })
+
+    const platformConfigDeployment = new BucketDeployment(this, 'PlatformConfigDeployment', {
+      sources: [Source.jsonData('platform-config.json', props.config)],
+      destinationBucket: props.bucket
+    })
+
     const role = new Role(this, 'role', {
       assumedBy: new ServicePrincipal('ec2.amazonaws.com')
     })
@@ -95,7 +99,7 @@ class ContainerServer extends Construct {
             'route53:ChangeResourceRecordSets'
           ],
           resources: [
-            `arn:aws:route53:::hostedzone/${hostedZone.hostedZoneId}`,
+            `arn:aws:route53:::hostedzone/${props.hostedZone.hostedZoneId}`,
             'arn:aws:route53:::change/*'
           ]
         }),
@@ -127,7 +131,7 @@ class ContainerServer extends Construct {
     )
 
     userData.addS3DownloadCommand({
-      bucket: props.platformConfigBucket,
+      bucket: props.bucket,
       bucketKey: 'platform-config.json',
       localFile: '/home/ubuntu/platform-config.json'
     })
@@ -152,25 +156,19 @@ class ContainerServer extends Construct {
       machineImage: new GenericLinuxImage({
         [SupportedRegions.North]: InstanceImages.UBUNTU
       }),
-      keyName: props.platformConfig.awsKeyPairName
+      keyName: props.config.awsKeyPairName
     })
 
-    props.platformConfigBucket.grantRead(ec2Instance, '*')
-    props.deploymentQueue.grant(ec2Instance, '*')
-
-    // const hostedZone = !!props.createNewHostedZone
-    //   ? new PublicHostedZone(this, 'HostedZone', { zoneName: props.domain })
-    //   : HostedZone.fromLookup(this, 'HostedZone', { domainName: props.domain });
+    props.bucket.grantRead(ec2Instance, '*')
+    deploymentQueue.grant(ec2Instance, '*')
 
-    // eslint-disable-next-line no-new
     new ARecord(this, 'WildcardRecord', {
-      zone: hostedZone,
-      recordName: `*.${props.platformConfig.activeEnvironment}.${props.platformConfig.domain}`,
+      zone: props.hostedZone,
+      recordName: `*.${props.config.environmentType}.${props.config.domain}`,
       target: RecordTarget.fromIpAddresses(ec2Instance.instancePublicIp)
     })
 
-    // eslint-disable-next-line no-new
-    new CfnOutput(this, 'Intance Ip', {
+    new CfnOutput(this, 'Instance Ip', {
       value: ec2Instance.instancePublicIp,
       description: 'Public ip of the ec2 instance',
       exportName: 'instanceIp'

package/src/infrastructure/deployment-target-stack.js

@@ -0,0 +1,47 @@
+/* eslint-disable no-new */
+const { nanoid } = require('nanoid')
+const { Stack, Duration, RemovalPolicy } = require('aws-cdk-lib')
+const { HostedZone, ARecord, RecordTarget } = require('aws-cdk-lib/aws-route53')
+const { Bucket, BucketEncryption, BlockPublicAccess } = require('aws-cdk-lib/aws-s3')
+const { ContainerServer } = require('./container-server')
+
+/**
+  * @class
+*/
+class DeploymentTargetStack extends Stack {
+  constructor(scope, id, props) {
+    super(scope, id, props)
+
+    if (!props?.config) {
+      throw ('[DeploymentTargetStack] No template provided')
+    }
+
+    const domain = `${props.config.environmentType}.${props.config.domain}`
+    const bucketId = nanoid().toLowerCase().replaceAll('_', '').replaceAll('-', '')
+    const bucketName = `${props.config.platformName}-${props.config.environmentType}-${bucketId}`
+
+    const hostedZone = new HostedZone(this, 'HostedZone', { zoneName: domain })
+
+    // TODO: this should probably not be destroyed....
+    const staticDeploymentTarget = new Bucket(this, 'StaticDeploymentTarget', {
+      bucketName: bucketName,
+      blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
+      encryption: BucketEncryption.S3_MANAGED,
+      removalPolicy: RemovalPolicy.DESTROY,
+      autoDeleteObjects: true
+    })
+
+    // Todo rename to container deployment target
+    // TODO: add persistant storage
+    new ContainerServer(this, 'ContainerDeploymentTarget', {
+      config: props.config,
+      hostedZone: hostedZone,
+      bucket: staticDeploymentTarget
+    })
+
+  }
+}
+
+module.exports = {
+  DeploymentTargetStack
+}

package/src/infrastructure/{establish-trust-stack.js → trust-ci-stack.js}

@@ -11,7 +11,7 @@ const {
 /**
   * @class
 */
-class EstablishTrustStack extends Stack {
+class TrustCiStack extends Stack {
   /**
    * Establishes trust between GithHub and Amazon Web Services.
    * This is needed so that we can interact with Amazon Web Services through
@@ -64,5 +64,5 @@ class EstablishTrustStack extends Stack {
 }
 
 module.exports = {
-  EstablishTrustStack
+  TrustCiStack
 }

package/src/template/platform-template.js

@@ -7,8 +7,7 @@ const { logError, logInfo } = require('../log')
  * @typedef {Object} PlatformTemplate
  * @property {string} platformName - Name of platform
  * @property {string} domain - example.com
- * @property {string[]} supportedDeploymentTypes - container
- * @property {string[]} supportedEnvironments - qa
+ * @property {string} environmentType - local, test, qa, prod
  *
  * @property {string} awsAccountId - Aws account id
  * @property {string=} awsRegion - ?
@@ -16,10 +15,6 @@ const { logError, logInfo } = require('../log')
  * @property {string} awsRoleToAssume - ?
  * @property {string=} awsDeploymentSqsArn - ?
  *
- * @property {string=} ciSubDomain - ?
- * @property {string|mumber=} ciInternalServerPort - ? | number;
- * @property {string=} ciServerName - ?
- * @property {string=} ciDockerNetworkName - ?
  * @property {string} ciGithubActionsRepo - organisation/repoName
  */
 

package/src/infrastructure/platform-stack.js

@@ -1,58 +0,0 @@
-/* eslint-disable no-new */
-const { Stack, Duration, RemovalPolicy } = require('aws-cdk-lib')
-const { Queue } = require('aws-cdk-lib/aws-sqs')
-const { Bucket, BucketEncryption, BlockPublicAccess } = require('aws-cdk-lib/aws-s3')
-const { Source, BucketDeployment } = require('aws-cdk-lib/aws-s3-deployment')
-const { nanoid } = require('nanoid')
-
-const { ContainerServer } = require('./container-server')
-
-const { SupportedDeploymentTypes } = require('../config')
-
-/**
-  * @class
-*/
-class PlatformStack extends Stack {
-  constructor(scope, id, props) {
-    super(scope, id, props)
-
-    if (!props?.config) {
-      throw ('[PlatformStack] No template config provided')
-    }
-
-    const isContainerDeploymentsEnabled = props.config.supportedDeploymentTypes.includes(SupportedDeploymentTypes.Container)
-
-    if (isContainerDeploymentsEnabled) {
-
-      const deploymentQueue = new Queue(this, 'DeploymentQueue', {
-        queueName: `${props.config.platformName}-${props.config.activeEnvironment}`,
-        receiveMessageWaitTime: Duration.seconds(20)
-      })
-
-      const platformConfigBucket = new Bucket(this, 'PlatformConfig', {
-        bucketName: `${props.config.platformName}-${props.config.activeEnvironment}-${nanoid().toLowerCase().replaceAll('_', '').replaceAll('-', '')}`,
-        blockPublicAccess: BlockPublicAccess.BLOCK_ALL,
-        encryption: BucketEncryption.S3_MANAGED,
-        removalPolicy: RemovalPolicy.DESTROY,
-        autoDeleteObjects: true
-      })
-
-      const platformConfigDeployment = new BucketDeployment(this, 'PlatformConfigDeployment', {
-        sources: [Source.jsonData('platform-config.json', props.config)],
-        destinationBucket: platformConfigBucket
-      })
-
-      new ContainerServer(this, 'ContainerServer', {
-        platformConfig: props.config,
-        platformConfigBucket: platformConfigDeployment.deployedBucket,
-        deploymentQueue: deploymentQueue
-      })
-
-    }
-
-  }
-}
-
-module.exports = {
-  PlatformStack
-}