aws_stack_builder 0.1.0

data/lib/generators/aws_stack_builder/templates/aws/geradores/ami-userdata.sh.template

@@ -0,0 +1,54 @@
+#!/bin/bash
+# create log file set -x
+exec > >(tee /tmp/user-data.log|logger -t user-data ) 2>&1
+# update OS
+sudo yum -y update
+# create user deploy
+sudo useradd deploy
+sudo bash -c "echo -e \"deploy ALL=(ALL) NOPASSWD: ALL\" >> /etc/sudoers"
+# install rvm and ruby
+sudo yum install -y gcc openssl-devel libyaml libyaml-devel libffi-devel libreadline readline-devel zlib1g zlib-devel gdbm-devel ncurses ncurses-devel ruby-devel gcc-c++ jq git patch libtool bison build-essential libc6 libgdbm
+sudo su - deploy -c "curl -sSL https://rvm.io/mpapis.asc | gpg2 --import -"
+sudo su - deploy -c "curl -sSL https://get.rvm.io | bash -s stable"
+sudo su - deploy -c "yes | rvm install 2.2.9"
+# install bundler
+sudo su - deploy -c "gem install bundler"
+# create ~/app and install application dependencies
+sudo su - deploy -c "mkdir -p ~/app/current"
+#  - java
+sudo yum -y install java-1.7.0-openjdk.x86_64
+sudo yum -y install java-1.7.0-openjdk-devel
+sudo su - deploy -c "echo \"export JAVA_HOME=/usr/lib/jvm/java-1.7.0\" >> ~/.bashrc"
+#  - libs needed by charlock holmes
+sudo yum -y install icu libicu libicu-devel xmlstarlet
+#  - mysql
+sudo yum -y install mysql-devel
+sudo su - deploy -c "echo \"export MYSQL_PASSWORD=taxweb2016\" >> ~/.bashrc"
+sudo su - deploy -c "echo \"export TAXRULES_PORTAL_DB=mysql\" >> ~/.bashrc"
+# install apache
+sudo yum -y install glibc.i686
+sudo yum -y install curl-devel httpd-devel-2.4.34
+sudo yum -y install httpd-2.4.34
+# execute <%= @app_name %> app_update.sh
+sudo su - deploy -c "cd ~/app/current; wget http://taxweb-deploy.s3.amazonaws.com/<%= @app_name %>/app_update.sh -O app_update.sh >/dev/null 2>&1"
+sudo su - deploy -c "cd ~/app/current && chmod 755 app_update.sh && ./app_update.sh staging web <%= @app_name %>"
+# set RAILS_ENV
+sudo su - deploy -c "echo \"export RAILS_ENV=staging\" >> ~/.bashrc"
+# install passenger
+sudo su - deploy -c "cd ~/app/current && gem install passenger -v 5.3.4 --no-rdoc --no-ri"
+sudo su - deploy -c "cd ~/app/current && passenger-install-apache2-module --languages ruby --auto"
+sudo su - deploy -c "cd ~/app/current && passenger-install-apache2-module --snippet > /tmp/passenger.conf"
+sudo su - deploy -c "echo \"\" >> /tmp/passenger.conf"
+sudo su - deploy -c "echo \"PassengerMaxRequestQueueSize 50\" >> /tmp/passenger.conf"
+sudo su - deploy -c "echo \"PassengerMaxPoolSize 15\" >> /tmp/passenger.conf"
+sudo su - deploy -c "sudo mv /tmp/passenger.conf /etc/httpd/conf.d/00passenger.conf"
+# config apache
+sudo su - -c "cd /etc/httpd/conf.d/ && rm <%= @app_name %>_app.conf; wget http://taxweb-deploy.s3.amazonaws.com/<%= @app_name %>/<%= @app_name %>_app.conf -O <%= @app_name %>_app.conf"
+# file system permissions
+sudo chmod 755 /home
+sudo chmod 755 /home/deploy
+sudo chmod -R 755 /home/deploy/app
+# restart apache
+sudo service httpd restart
+# finished
+echo "Instance configuration finished!"

data/lib/generators/aws_stack_builder/templates/aws/geradores/build_templates.sh.template

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+python cf-base-instance.py <%= @app_name %>
+python cf-stack-generator.py staging <%= @app_name %>
+python cf-stack-generator.py production <%= @app_name %>
+mkdir -p ../cloud_formation_templates
+mv -f *.json ../cloud_formation_templates

data/lib/generators/aws_stack_builder/templates/aws/geradores/cf-base-instance.py.template

@@ -0,0 +1,96 @@
+from troposphere import Base64, Join, Parameter, Ref, Template, Tags
+import troposphere.ec2 as ec2
+
+# template
+template = Template()
+template.add_version("2010-09-09")
+template.add_description("AWS CloudFormation template for base instance")
+
+# parameters
+subnet_id = Parameter(
+    "SubnetId",
+    Type="AWS::EC2::Subnet::Id",
+    Default="subnet-0b3b0d6e",
+    Description="Id of the Subnet which will hold the instance"
+)
+template.add_parameter(subnet_id)
+
+vpc_id = Parameter(
+    "VpcId",
+    Type="AWS::EC2::VPC::Id",
+    Default="vpc-ebe0c38e",
+    Description="Id of the VPC which will hold the instance"
+)
+template.add_parameter(vpc_id)
+
+instance_profile = Parameter(
+    "InstanceProfileName",
+    Type="String",
+    Default="<%= @app_name %>",
+    Description="Name of the instance profile to associate with the instance"
+)
+template.add_parameter(instance_profile)
+
+
+# security group
+sg = ec2.SecurityGroup(
+    '<%= @app_name %>SG',
+    GroupDescription="Security Group for base instance.",
+    Tags=Tags(
+       Name='<%= @app_name %>-base-instance-sg',
+       Custo='<%= @app_name %>'
+    ),
+    SecurityGroupIngress=[
+        ec2.SecurityGroupRule(
+            IpProtocol="tcp",
+            FromPort="22",
+            ToPort="22",
+            CidrIp="0.0.0.0/0"
+        ),
+        ec2.SecurityGroupRule(
+            IpProtocol="tcp",
+            FromPort="9090",
+            ToPort="9090",
+            CidrIp="0.0.0.0/0"
+        ),
+        ec2.SecurityGroupRule(
+            IpProtocol="tcp",
+            FromPort="80",
+            ToPort="80",
+            CidrIp="0.0.0.0/0"
+        ),
+        ec2.SecurityGroupRule(
+            IpProtocol="tcp",
+            FromPort="443",
+            ToPort="443",
+            CidrIp="0.0.0.0/0"
+        )
+    ],
+    VpcId=Ref(vpc_id)
+)
+template.add_resource(sg)
+
+userdata_file = open("ami-userdata.sh", "r")
+userdata = userdata_file.read()
+
+# instance
+instance = ec2.Instance(
+    "<%= @app_name %>BaseInstance",
+    Tags=Tags(
+        Name='<%= @app_name %>-Base-Instance',
+        Custo='<%= @app_name %>'
+    ),
+    ImageId="ami-a9d09ed1",
+    InstanceType="t3.medium",
+    KeyName="taxweb-AWS-US-West",
+    SubnetId=Ref(subnet_id),
+    SecurityGroupIds=[Ref(sg)],
+    #IamInstanceProfile=Ref(instance_profile),
+    UserData=Base64(userdata)
+)
+template.add_resource(instance)
+
+
+f = open("cf-base-instance.json", "w+")
+f.write(template.to_json())
+f.close()

data/lib/generators/aws_stack_builder/templates/aws/geradores/cf-stack-generator.py.template

@@ -0,0 +1,456 @@
+from troposphere import Base64, Join, GetAtt, Tags, Parameter, Ref, Template
+from troposphere.autoscaling import AutoScalingGroup, Tag, LaunchConfiguration
+from troposphere.elasticloadbalancing import LoadBalancer
+from troposphere.policies import AutoScalingReplacingUpdate, AutoScalingRollingUpdate, UpdatePolicy
+from troposphere.route53 import RecordSetType
+from troposphere.ec2 import SecurityGroupIngress
+
+import troposphere.ec2 as ec2
+import troposphere.elasticloadbalancingv2 as elb
+
+# global variables
+# python parameters
+app_name = 'taxrules'
+env = 'staging'
+# derived variables
+stack_name = None
+stack_name_strict = None
+
+
+# get user inputs
+def define_python_parameters():
+    # input params
+    global app_name
+    default = app_name
+    app_name = raw_input('What is the application name? [' + default + '] ') or default
+
+    global env
+    default = env
+    env = raw_input('What is the environment (production/staging)? [' + default + '] ') or default
+
+    # derived variables 
+    env_suffix = 'Hml' if env == 'staging' else ''
+    global stack_name
+    stack_name = app_name.lower().replace(' ', '-')
+    if env_suffix:
+        stack_name = stack_name + "-" + env_suffix.lower()
+
+    global stack_name_strict
+    stack_name_strict = app_name.replace(' ', '').replace('-', '').replace('_', '') + env_suffix
+
+
+def define_cloud_formation_parameters(template):
+    db_security_group_id = Parameter(
+        "DbSecurityGroupId",
+        Type="AWS::EC2::SecurityGroup::Id",
+        Default="sg-0f84c5afcb16d79fe",
+        Description="The id of the security group of the database (database is not part of the stack created). The resources of the stack will be given access to the DB port in this security group.",
+    )
+    template.add_parameter(db_security_group_id)
+
+    db_port = Parameter(
+        "DbPort",
+        Type="Number",
+        Default="3306",
+        Description="The port where the DB listens for incoming connections. The resources of the stack will be given access to this port in the DB security group.",
+    )
+    template.add_parameter(db_port)
+
+    ssl_certificate_arn = Parameter(
+        "SSLCertificateARN",
+        Type="String",
+        Default="arn:aws:acm:us-west-2:676854543053:certificate/ab8afefd-5f93-48c2-a3fa-ae5a8e5f99e6",
+        Description="ARN of the SSL certificate for load balancer."
+    )
+    template.add_parameter(ssl_certificate_arn)
+
+    vpc_id = Parameter(
+        "VpcId",
+        Type="AWS::EC2::VPC::Id",
+        Default="vpc-ebe0c38e",
+        Description="Id of the VPC which will hold the resources"
+    )
+    template.add_parameter(vpc_id)
+
+    subnet_id1 = Parameter(
+        "SubnetId1",
+        Type="AWS::EC2::Subnet::Id",
+        Default="subnet-0b3b0d6e",
+        Description="Id of the 1st Subnet which will hold the resources"
+    )
+    template.add_parameter(subnet_id1)
+
+    subnet_id2 = Parameter(
+        "SubnetId2",
+        Type="AWS::EC2::Subnet::Id",
+        Default="subnet-2f267258",
+        Description="Id of the 2nd Subnet which will hold the resources"
+    )
+    template.add_parameter(subnet_id2)
+
+    alb_health_check_path = Parameter(
+        "AlbHealthCheckPath",
+        Type="String",
+        Default="/authentication/sign_in",
+        Description="Ping path destination used by the load balancer to check server health"
+    )
+    template.add_parameter(alb_health_check_path)
+
+    base_ami = Parameter(
+        "BaseAMI",
+        Type="AWS::EC2::Image::Id",
+        Default="ami-025ff841a2680ff81",
+        Description="Id of the base AMI"
+    )
+    template.add_parameter(base_ami)
+
+    instance_type = Parameter(
+        "InstanceType",
+        Type="String",
+        Default="t2.small",
+        Description="Type of the instance to be used."
+    )
+    template.add_parameter(instance_type)
+
+    instance_profile = Parameter(
+        "InstanceProfileName",
+        Type="String",
+        Default="<%= @app_name %>",
+        Description="Name of the instance profile to associate with the instance"
+    )
+    template.add_parameter(instance_profile)
+
+    return {"db_security_group_id": db_security_group_id,
+            "db_port": db_port,
+            "ssl_certificate_arn": ssl_certificate_arn,
+            "vpc_id": vpc_id,
+            "subnet_id1": subnet_id1,
+            "subnet_id2": subnet_id2,
+            "alb_health_check_path": alb_health_check_path,
+            "base_ami": base_ami,
+            "instance_type": instance_type,
+            "instance_profile": instance_profile}
+
+
+def define_security_group(template, vpc_id):
+    sg = ec2.SecurityGroup(
+        stack_name_strict + 'SG',
+        GroupDescription="Security Group for " + stack_name + " stack.",
+        Tags=Tags(
+            Name=stack_name,
+            Custo=app_name
+        ),
+        SecurityGroupIngress=[
+            ec2.SecurityGroupRule(
+                IpProtocol="tcp",
+                FromPort="22",
+                ToPort="22",
+                CidrIp="0.0.0.0/0"
+            ),
+            ec2.SecurityGroupRule(
+                IpProtocol="tcp",
+                FromPort="9090",
+                ToPort="9090",
+                CidrIp="0.0.0.0/0"
+            ),
+            ec2.SecurityGroupRule(
+                IpProtocol="tcp",
+                FromPort="80",
+                ToPort="80",
+                CidrIp="0.0.0.0/0"
+            ),
+            ec2.SecurityGroupRule(
+                IpProtocol="tcp",
+                FromPort="443",
+                ToPort="443",
+                CidrIp="0.0.0.0/0"
+            )
+        ],
+        VpcId=Ref(vpc_id)
+    )
+    template.add_resource(sg)
+    return sg
+
+
+def define_load_balancer(template, 
+	sg, 
+	alb_health_check_path, 
+	vpc_id, 
+	subnet_id1, 
+	subnet_id2, 
+	ssl_certificate_arn):
+    alb_target_group_80 = elb.TargetGroup(
+        stack_name_strict + "TG80",
+        Tags=Tags(
+            Name=stack_name,
+            Custo=app_name
+        ),
+        HealthCheckPath=Ref(alb_health_check_path),
+        HealthCheckIntervalSeconds="30",
+        HealthCheckProtocol="HTTP",
+        HealthCheckTimeoutSeconds="30",
+        HealthyThresholdCount="3",
+        Matcher=elb.Matcher(
+            HttpCode="200,301"),
+        Port=80,
+        Protocol="HTTP",
+        UnhealthyThresholdCount="2",
+        VpcId=Ref(vpc_id)
+    )
+    template.add_resource(alb_target_group_80)
+
+    alb_target_group_9090 = elb.TargetGroup(
+        stack_name_strict + "TG9090",
+        Tags=Tags(
+            Name=stack_name,
+            Custo=app_name
+        ),
+        HealthCheckPath=Ref(alb_health_check_path),
+        HealthCheckIntervalSeconds="30",
+        HealthCheckProtocol="HTTP",
+        HealthCheckTimeoutSeconds="30",
+        HealthyThresholdCount="3",
+        Matcher=elb.Matcher(
+            HttpCode="200,301"),
+        Port=9090,
+        Protocol="HTTP",
+        UnhealthyThresholdCount="2",
+        VpcId=Ref(vpc_id)
+    )
+    template.add_resource(alb_target_group_9090)
+
+    alb = elb.LoadBalancer(
+        stack_name_strict + "ALB",
+        Tags=Tags(
+            Name=stack_name,
+            Custo=app_name
+        ),
+        Scheme="internet-facing",
+        Subnets=[Ref(subnet_id1), Ref(subnet_id2)],
+        SecurityGroups=[Ref(sg)]
+    )
+    template.add_resource(alb)
+
+    alb_listener_80 = elb.Listener(
+        stack_name_strict + "ListenerALB80",
+        Port=80,
+        Protocol="HTTP",
+        LoadBalancerArn=Ref(alb),
+        DefaultActions=[elb.Action(
+            Type="forward",
+            TargetGroupArn=Ref(alb_target_group_80)
+        )]
+    )
+    template.add_resource(alb_listener_80)
+
+    alb_listener_443 = elb.Listener(
+        stack_name_strict + "ListenerALB443",
+        Port=443,
+        Protocol="HTTPS",
+        Certificates=[elb.Certificate(
+            CertificateArn=Ref(ssl_certificate_arn)
+        )],
+        LoadBalancerArn=Ref(alb),
+        DefaultActions=[elb.Action(
+            Type="forward",
+            TargetGroupArn=Ref(alb_target_group_9090)
+        )]
+    )
+    template.add_resource(alb_listener_443)
+
+    return {"loadbalancer": alb,
+            "alb_target_group_80": alb_target_group_80,
+            "alb_target_group_9090": alb_target_group_9090,
+            "alb_listener_80": alb_listener_80,
+            "alb_listener_9090": alb_listener_443}
+
+
+def define_web_auto_scaling(template, 
+        base_ami, 
+        instance_type, 
+        instance_profile, 
+        subnet_id1, 
+        subnet_id2, 
+        sg,
+        alb_target_group_80, 
+        alb_target_group_9090):
+    web_launch_config = LaunchConfiguration(
+        stack_name_strict + "WebLC",
+        UserData=Base64(Join('', [
+            '#!/bin/bash\n',
+            'set -x\n',
+            'exec > >(tee /tmp/user-data.log|logger -t user-data ) 2>&1\n',
+            'sudo su - deploy -c "echo \\"export RAILS_ENV=' + env + '\\" >> ~/.bashrc"\n',
+            'sudo su - deploy -c "cd ~/app/current; wget http://taxweb-deploy.s3.amazonaws.com/' + app_name  + '/app_update.sh -O app_update.sh >/dev/null 2>&1"\n',
+            'sudo su - deploy -c "cd ~/app/current && chmod 755 app_update.sh && ./app_update.sh ' + env + ' web ' + app_name + '"\n'
+        ])),
+        ImageId=Ref(base_ami),
+        InstanceType=Ref(instance_type),
+        IamInstanceProfile=Ref(instance_profile),
+        KeyName="taxweb-AWS-US-West",
+        SecurityGroups=[Ref(sg)]
+    )
+    template.add_resource(web_launch_config)
+
+    web_autoscaling_group = AutoScalingGroup(
+        stack_name_strict + "WebASG",
+        Tags=[
+            Tag("Name", stack_name + "-web", True),
+            Tag("Custo", app_name, True),
+            Tag("Env", env, True),
+            Tag("Role", "web", True),
+        ],
+        LaunchConfigurationName=Ref(web_launch_config),
+        MinSize=1,
+        MaxSize=1,
+        DesiredCapacity=1,
+        VPCZoneIdentifier=[Ref(subnet_id1), Ref(subnet_id2)],
+        TargetGroupARNs=[Ref(alb_target_group_80), Ref(alb_target_group_9090)],
+        HealthCheckType="ELB",
+        HealthCheckGracePeriod="300",
+    )
+    template.add_resource(web_autoscaling_group)
+
+    return {"launch_config": web_launch_config,
+            "autoscaling_group": web_autoscaling_group}
+
+
+def define_auto_scaling(template,
+        base_ami,
+        instance_type,
+        instance_profile,
+        subnet_id1,
+        subnet_id2,
+        sg,
+        role,
+        initial_size):
+    role_name_strict = role.replace(' ', '').replace('-', '').replace('_', '')
+    launch_config = LaunchConfiguration(
+        stack_name_strict + role_name_strict.capitalize() + "LC",
+        UserData=Base64(Join('', [
+            '#!/bin/bash\n',
+            'set -x\n',
+            'exec > >(tee /tmp/user-data.log|logger -t user-data ) 2>&1\n',
+            'sudo su - deploy -c "echo \\"export RAILS_ENV=' + env + '\\" >> ~/.bashrc"\n',
+            'sudo su - deploy -c "cd ~/app/current; wget http://taxweb-deploy.s3.amazonaws.com/' + app_name + '/app_update.sh -O app_update.sh >/dev/null 2>&1"\n',
+            'sudo su - deploy -c "cd ~/app/current && chmod 755 app_update.sh && ./app_update.sh ' + env + ' ' + role + ' ' + app_name + '"\n'
+        ])),
+        ImageId=Ref(base_ami),
+        InstanceType=Ref(instance_type),
+        IamInstanceProfile=Ref(instance_profile),
+        KeyName="taxweb-AWS-US-West",
+        SecurityGroups=[Ref(sg)]
+    )
+    template.add_resource(launch_config)
+
+    autoscaling_group = AutoScalingGroup(
+        stack_name_strict + role_name_strict.capitalize() + "ASG",
+        Tags=[
+            Tag("Name", stack_name + "-" + role, True),
+            Tag("Custo", app_name, True),
+            Tag("Env", env, True),
+            Tag("Role", role, True),
+        ],
+        LaunchConfigurationName=Ref(launch_config),
+        MinSize=initial_size,
+        MaxSize=initial_size,
+        DesiredCapacity=initial_size,
+        VPCZoneIdentifier=[Ref(subnet_id1), Ref(subnet_id2)],
+        # TargetGroupARNs=[Ref(alb_target_group_80), Ref(alb_target_group_9090)],
+        HealthCheckType="EC2",
+        HealthCheckGracePeriod="300",
+    )
+    template.add_resource(autoscaling_group)
+
+    return {"launch_config": launch_config,
+            "autoscaling_group": autoscaling_group}
+
+
+def define_dns(template, alb):
+    route53_record = template.add_resource(RecordSetType(
+        stack_name_strict + "WebDNS",
+        HostedZoneName="taxweb.com.br.",
+        Name=stack_name + ".taxweb.com.br.",
+        ResourceRecords=[GetAtt(alb, "DNSName")],
+        TTL=60,
+        Type="CNAME"
+    ))
+
+    return route53_record
+
+
+def add_ingress_rule_to_db_security_group(template, db_sg, db_port, stack_sg):
+    sg_ingress = SecurityGroupIngress(
+            stack_name_strict + "DbIngressRule",
+            IpProtocol="tcp",
+            FromPort=Ref(db_port),
+            ToPort=Ref(db_port),
+            SourceSecurityGroupId=Ref(stack_sg),
+            GroupId=Ref(db_sg)
+    )
+    template.add_resource(sg_ingress)
+
+
+def gerador():
+    # template
+    template = Template()
+    template.add_version("2010-09-09")
+    template.add_description("AWS CloudFormation template for Taxweb RoR applications")
+
+    # python parameters
+    define_python_parameters()
+
+    # parameters
+    parameters = define_cloud_formation_parameters(template)
+
+    # security group
+    sg = define_security_group(template, parameters["vpc_id"])
+
+    # ALB (Application Load Balancer)
+    alb = define_load_balancer(template, 
+		sg, 
+		parameters["alb_health_check_path"], 
+		parameters["vpc_id"], 
+		parameters["subnet_id1"], 
+		parameters["subnet_id2"],
+		parameters["ssl_certificate_arn"])
+
+    # web auto scaling
+    define_web_auto_scaling(template, 
+            parameters["base_ami"], 
+            parameters["instance_type"], 
+            parameters["instance_profile"], 
+            parameters["subnet_id1"], 
+            parameters["subnet_id2"],
+            sg,
+            alb["alb_target_group_80"], 
+            alb["alb_target_group_9090"])
+
+    <%- @roles.each do |role| %>
+    # worker-relatorio auto scaling
+    define_auto_scaling(template,
+            parameters["base_ami"],
+            parameters["instance_type"],
+            parameters["instance_profile"],
+            parameters["subnet_id1"],
+            parameters["subnet_id2"],
+            sg,
+            "worker-<%= role %>",
+            1)
+    <% end %>
+
+    # DNS
+    define_dns(template, alb["loadbalancer"])
+
+    # Add ingress rule to db security group
+    add_ingress_rule_to_db_security_group(template, parameters["db_security_group_id"], parameters["db_port"], sg)
+
+    final_json = template.to_json()
+
+    f = open(stack_name + "-stack-cf.json", "w+")
+    f.write(final_json)
+    f.close()
+
+
+# Chammo o Gerador do Arquivo
+gerador()