stacco 0.1.36 → 0.1.37

data/priv/layers/client-api.json

@@ -131,19 +131,6 @@
     "KeyName": {"Ref": "IAMKeypairNameVar"},
     "SecurityGroups": [{"Ref": "FrontendSecurityGroup"}],
 
-    "BlockDeviceMappings": [
-      {"DeviceName": "/dev/xvdc", "Ebs": {
-        "SnapshotId": {"Ref": "DockerLibrarySnapshotVar"},
-        "VolumeSize": "50",
-        "VolumeType" : "gp2"
-      }},
-
-      {"DeviceName": "/dev/xvdd", "Ebs": {
-        "VolumeSize": "300",
-        "VolumeType" : "gp2"
-      }}
-    ],
-
     "UserData": {"Fn::Base64": {"Fn::Join": ["", [
       "#!/bin/bash\n",
       "export AWS_REGION='", {"Ref": "AWS::Region"}, "'\n",

data/priv/layers/db.json

@@ -1,62 +1,89 @@
 {
   "Resources": {
 
-    "DBSubnet" : {
-      "Type" : "AWS::EC2::Subnet",
-      "Properties" : {
-        "VpcId" : { "Ref" : "VPC" },
-        "CidrBlock" : { "Fn::FindInMap" : [ "SubnetConfig", "DB", "CIDR" ]},
-        "AvailabilityZone" : {"Fn::Join": ["", [{"Ref": "AWS::Region"}, { "Fn::FindInMap" : [ "SubnetConfig", "DB", "AvailabilityZone" ]}]]}
+    "DBSubnet": {
+      "Type": "AWS::EC2::Subnet",
+      "Properties": {
+        "VpcId": {"Ref": "VPC"},
+        "CidrBlock": {"Fn::FindInMap": [ "SubnetConfig", "DB", "CIDR" ]},
+        "AvailabilityZone": {"Fn::Join": ["", [{"Ref": "AWS::Region"}, {"Fn::FindInMap": [ "SubnetConfig", "DB", "AvailabilityZone" ]}]]}
       }
     },
 
-    "DBSubnetRouteTableAssociation" : {
-      "Type" : "AWS::EC2::SubnetRouteTableAssociation",
-      "Properties" : {
-        "SubnetId" : { "Ref" : "DBSubnet" },
-        "RouteTableId" : { "Ref" : "PrivateRouteTable" }
+    "DBSubnetRouteTableAssociation": {
+      "Type": "AWS::EC2::SubnetRouteTableAssociation",
+      "Properties": {
+        "SubnetId": {"Ref": "DBSubnet"},
+        "RouteTableId": {"Ref": "PrivateRouteTable"}
       }
     },
 
-    "DBSubnetNetworkAclAssociation" : {
-      "Type" : "AWS::EC2::SubnetNetworkAclAssociation",
-      "Properties" : {
-        "SubnetId" : { "Ref" : "DBSubnet" },
-        "NetworkAclId" : { "Ref" : "PrivateNetworkAcl" }
+    "DBSubnetNetworkAclAssociation": {
+      "Type": "AWS::EC2::SubnetNetworkAclAssociation",
+      "Properties": {
+        "SubnetId": {"Ref": "DBSubnet"},
+        "NetworkAclId": {"Ref": "PrivateNetworkAcl"}
       }
     },
 
-    "DB" : {
+    "DBConfig": {
+      "Type": "AWS::RDS::DBParameterGroup",
+
+      "Properties": {
+        "Description": "Stacco-controlled DB configuration",
+        "Family": "postgres9.3",
+
+        "Parameters": {
+          "client_min_messages": "warning",
+          "log_checkpoints": "0",
+          "log_duration": "0",
+          "log_filename": "postgresql.log.%Y-%m-%d",
+          "log_min_duration_statement": "500",
+          "log_min_messages": "info",
+          "log_rotation_age": "1440",
+          "log_rotation_size": "2097151",
+          "log_statement": "mod",
+          "log_statement_stats": "0",
+          "statement_timeout": "600000",
+          "track_activities": "1",
+          "track_counts": "1",
+          "track_io_timing": "1"
+        }
+      }
+    },
+
+    "DB": {
       "Type": "AWS::RDS::DBInstance",
-      "DeletionPolicy" : "Snapshot",
+      "DeletionPolicy": "Snapshot",
       "Properties": {
         "DBInstanceClass": "db.m3.medium",
         "MultiAZ": "true",
         "DBSubnetGroupName": {"Ref": "DBSubnets"},
-        "VPCSecurityGroups": [{ "Ref" : "DBSecurityGroup" }],
+        "VPCSecurityGroups": [{"Ref": "DBSecurityGroup"}],
 
         "AllocatedStorage": "80",
         "Engine": "Postgres",
         "EngineVersion": "9.3.3",
+        "DBParameterGroupName": {"Ref": "DBConfig"},
 
-        "MasterUsername": { "Ref" : "DBAdminUsernameVar" },
-        "MasterUserPassword": { "Ref" : "DBAdminPasswordVar" }
+        "MasterUsername": {"Ref": "DBAdminUsernameVar"},
+        "MasterUserPassword": {"Ref": "DBAdminPasswordVar"}
       }
     },
 
     "DBSubnets": {
-      "Type" : "AWS::RDS::DBSubnetGroup",
-      "Properties" : {
-        "DBSubnetGroupDescription" : "Span Private and DB subnets for MultiAZ",
-        "SubnetIds" : [ {"Ref": "PrivateSubnet"}, {"Ref": "DBSubnet"} ]
+      "Type": "AWS::RDS::DBSubnetGroup",
+      "Properties": {
+        "DBSubnetGroupDescription": "Span Private and DB subnets for MultiAZ",
+        "SubnetIds": [ {"Ref": "PrivateSubnet"}, {"Ref": "DBSubnet"} ]
       }
     },
 
-    "DBSecurityGroup" : {
-      "Type" : "AWS::EC2::SecurityGroup",
-      "Properties" : {
-        "GroupDescription" : "Database access",
-        "VpcId" : { "Ref" : "VPC" },
+    "DBSecurityGroup": {
+      "Type": "AWS::EC2::SecurityGroup",
+      "Properties": {
+        "GroupDescription": "Database access",
+        "VpcId": {"Ref": "VPC"},
         "SecurityGroupIngress": [],
         "SecurityGroupEgress": [
           {"IpProtocol": "-1", "CidrIp": "0.0.0.0/0"}

data/priv/layers/task-generate-base-image.json

@@ -32,7 +32,7 @@
   },
   "Properties" : {
     "InstanceType": "m3.large",
-    "ImageId": {"Ref": "InstanceAMIVar"},
+    "ImageId": {"Ref": "BaseAMIVar"},
     "KeyName": {"Ref": "IAMKeypairNameVar"},
 
     "NetworkInterfaces": [{
@@ -45,7 +45,12 @@
 
     "BlockDeviceMappings": [
       {"DeviceName": "/dev/xvdc", "Ebs": {
-        "VolumeSize": "5"
+        "VolumeSize": "5",
+        "VolumeType" : "gp2"
+      }},
+      {"DeviceName": "/dev/xvdd", "Ebs": {
+        "VolumeSize": "50",
+        "VolumeType" : "gp2"
       }}
     ],
 
@@ -56,8 +61,6 @@
       "export AWS_INSTANCE_LOGICAL_NAME='BaseImageGenerator'\n",
       "export AWS_INSTANCE_WAIT_HANDLE='", {"Ref": "BaseImageGeneratorReadyWaitHandle"}, "'\n",
       {"Ref": "UserDataEnvironmentVar"}, "\n",
-      {"Ref": "CommonRoleScriptVar"}, "\n",
-      {"Ref": "DockerHostRoleScriptVar"}, "\n",
       {"Ref": "BaseImageGeneratorRoleScriptVar"}, "\n"
     ]]}}
   }

data/priv/roles/Backend.sh

@@ -41,9 +41,15 @@ EOF
 chmod a+x /usr/local/sbin/bexng-before-start
 
 
-
+echo "starting docker operations"
 docker-utils bootstrap
 
+if [ "${WALLET_NETWORK}" = "testnet3" ]; then
+  docker run --name='bitcoind-data' --volumes-from='tbtc-data' tianon/true:latest
+else
+  docker run --name='bitcoind-data' --volumes-from='btc-data' tianon/true:latest
+fi
+
 docker-utils define-ambassador-service "postgresql" \
   "${DB_HOST}" "${DB_PORT}"
 
@@ -62,20 +68,6 @@ docker-utils define-service "bexio/bexng-frontend" \
   --dependency "bexng" \
   --publish="80:8080"
 
-
-echo "bootstrapping bitcoind service"
-start bitcoind
-stop bitcoind
-bitcoind_data_volume=$(docker inspect -f '{{.Volumes}}' bitcoind-data | tr '[]' '  ' | cut -d' ' -f 2 | cut -d':' -f 2)
-pushd "${bitcoind_data_volume}" >/dev/null
-  until aws --region="${AWS_REGION}" s3 sync --delete "s3://bex-blockchain-${WALLET_NETWORK}/" "./"; do
-    echo "error occurred in sync; retrying"
-    sleep 0.2
-  done
-
-  find . -name "*.gz" -exec gunzip -v \{\} \;
-popd >/dev/null
-
 echo "starting docker-container services..."
 start bitcoind
 start postgresql

data/priv/roles/BaseImageGenerator.sh

@@ -25,63 +25,42 @@ set -e
 export HOME=/root
 cd "$HOME"
 
-echo "ensuring internet connectivity..."
-curl -sL -I -o /dev/null --fail --show-error --retry 100 'http://bex-status.s3.amazonaws.com/status.json'
-echo "found an internet connection."
-
-echo "configuring hostname from aws metadata service"
-host_title="${AWS_STACK_NAME}-$(echo "${AWS_INSTANCE_LOGICAL_NAME}" | sed 's/LaunchConfiguration$//')"
-private_hostname=$(curl -sL http://169.254.169.254/latest/meta-data/local-hostname)
-short_private_hostname=$(echo "${private_hostname}" | cut -d'.' -f 1)
-echo "${host_title}" > /etc/hostname
-hostname -b -F /etc/hostname
-sed -i '/127\.0\./d' /etc/hosts
-cat >>/etc/hosts <<EOF
-127.0.0.1 localhost
-127.0.1.1 ${private_hostname} ${host_title} ${short_private_hostname}
-EOF
-
 echo "enabling mirroring for security apt sources"
 sed -i "s/security\.ubuntu\.com/${AWS_REGION}.ec2.archive.ubuntu.com/g" /etc/apt/sources.list
 
-echo "disablng kernel and initramfs updates"
-echo $(dpkg -l "*$(uname -r)*" | grep image | awk '{print $2}') hold | dpkg --set-selections
-sed -i 's/=yes/=no/g' /etc/initramfs-tools/update-initramfs.conf
-
 echo "adding vendor apt sources"
 apt-key adv --keyserver 'hkp://pgp.mit.edu:80' --recv-keys 'C43C79AD'
 echo "deb http://rep.logentries.com/ trusty main" > /etc/apt/sources.list.d/logentries.list
 
 echo "installing packages"
-apt-get clean
-apt-get update
-apt-get upgrade -qy
-apt-get install -qy python-setuptools python-pip python-setproctitle htop tree btrfs-tools xz-utils logentries bundler ruby-nokogiri
+afg() {
+  DEBIAN_FRONTEND=noninteractive \
+  apt-get -q -y --force-yes \
+  -o Dpkg::Options::="--force-confnew" \
+  $@
+}
+
+afg clean
+afg update
+afg dist-upgrade
+afg install htop tree btrfs-tools xz-utils logentries kexec-tools debconf-utils
+afg install python-setuptools python-pip python-setproctitle bundler ruby-nokogiri
+afg clean
+
+debconf-set-selections <<EOF
+kexec-tools kexec-tools/load_kexec boolean true
+kexec-tools kexec-tools/use_grub_config boolean true
+EOF
+
+
 pip install -q awscli
 gem install -q --no-rdoc --no-ri aws-sdk docker-utils
 
-if [ -n "$LOGENTRIES_ACCOUNT_KEY" ]; then
-  echo "starting remote logging"
-  host_key=$(docker-utils get-host-uuid "${AWS_INSTANCE_LOGICAL_NAME}.${AWS_STACK_NAME}")
-  le init --account-key="${LOGENTRIES_ACCOUNT_KEY}" --agent-key="${host_key}"
-  le register --force --name="${host_title}" --hostname="${private_hostname}"
-  apt-get install -qy logentries-daemon 2>/dev/null || :
-
-  logs_to_follow="syslog messages dmesg auth.log boot.log daemon.log dpkg.log kern.log cron secure faillog cloud-init-output.log"
-  for log_name in ${logs_to_follow}; do
-    log_path="/var/log/${log_name}"
-    if [ -f "${log_path}" ]; then
-      log_name=$(basename "${log_path}" '.log')
-      le follow --name="${log_name}" "${log_path}" 2>/dev/null 1>&2
-    fi
-  done
-
-  service logentries restart
-fi
+touch /etc/default/logentries_not_to_be_run
+apt-get install -qy logentries-daemon 2>/dev/null || :
 
 echo "installing aws cfn-tools"
-easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
-cfn-init --region="${AWS_REGION}" --stack="${AWS_STACK_NAME}" --resource="${AWS_INSTANCE_LOGICAL_NAME}" || die 'Failed to run cfn-init'
+easy_install "https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz"
 
 docker-utils bootstrap
 

data/priv/roles/Common.sh

@@ -6,6 +6,7 @@ _exit_handler(){
     echo "Stacco cloud-init handler finished"
     cfn-signal -s true "${AWS_INSTANCE_WAIT_HANDLE}"
   else
+    sleep 1000
     echo "Stacco cloud-init handler aborted (${exit_code}: ${_exit_handler_reason})"
     cfn-signal -e "${exit_code}" -r "${_exit_handler_reason}" "${AWS_INSTANCE_WAIT_HANDLE}"
   fi
@@ -13,24 +14,11 @@ _exit_handler(){
   exit "${exit_code}"
 }
 trap _exit_handler EXIT
-
-die(){
-  _exit_handler_reason="$1"
-  echo "error: $1" >&2
-  exit 1
-}
-
 set -e
 
-
-
 export HOME=/root
 cd "$HOME"
 
-echo "ensuring internet connectivity..."
-curl -sL -I -o /dev/null --fail --show-error --retry 100 'http://bex-status.s3.amazonaws.com/status.json'
-echo "found an internet connection."
-
 echo "configuring hostname from aws metadata service"
 host_title="${AWS_STACK_NAME}-$(echo "${AWS_INSTANCE_LOGICAL_NAME}" | sed 's/LaunchConfiguration$//')"
 private_hostname=$(curl -sL http://169.254.169.254/latest/meta-data/local-hostname)
@@ -43,45 +31,26 @@ cat >>/etc/hosts <<EOF
 127.0.1.1 ${private_hostname} ${host_title} ${short_private_hostname}
 EOF
 
-echo "enabling mirroring for security apt sources"
-sed -i "s/security\.ubuntu\.com/${AWS_REGION}.ec2.archive.ubuntu.com/g" /etc/apt/sources.list
-
-echo "disablng kernel and initramfs updates"
-echo $(dpkg -l "*$(uname -r)*" | grep image | awk '{print $2}') hold | dpkg --set-selections
-sed -i 's/=yes/=no/g' /etc/initramfs-tools/update-initramfs.conf
-
-echo "adding vendor apt sources"
-apt-key adv --keyserver 'hkp://pgp.mit.edu:80' --recv-keys 'C43C79AD'
-echo "deb http://rep.logentries.com/ trusty main" > /etc/apt/sources.list.d/logentries.list
-
-echo "installing packages"
-apt-get clean
-apt-get update
-apt-get upgrade -qy
-apt-get install -qy python-setuptools python-pip python-setproctitle htop tree btrfs-tools xz-utils logentries bundler ruby-nokogiri
-pip install -q awscli
-gem install -q --no-rdoc --no-ri aws-sdk docker-utils
 
 if [ -n "$LOGENTRIES_ACCOUNT_KEY" ]; then
+  le clean
+  rm -f /etc/default/logentries_not_to_be_run
+
   echo "starting remote logging"
   host_key=$(docker-utils get-host-uuid "${AWS_INSTANCE_LOGICAL_NAME}.${AWS_STACK_NAME}")
+
   le init --account-key="${LOGENTRIES_ACCOUNT_KEY}" --agent-key="${host_key}"
   le register --force --name="${host_title}" --hostname="${private_hostname}"
-  apt-get install -qy logentries-daemon 2>/dev/null || :
+  service logentries start
 
-  logs_to_follow="syslog messages dmesg auth.log boot.log daemon.log dpkg.log kern.log cron secure faillog cloud-init-output.log"
-  for log_name in ${logs_to_follow}; do
+  logs_to_follow=(syslog messages dmesg auth.log boot.log daemon.log dpkg.log kern.log cron secure faillog cloud-init-output.log)
+  for log_name in "${logs_to_follow[@]}"; do
     log_path="/var/log/${log_name}"
     if [ -f "${log_path}" ]; then
       log_name=$(basename "${log_path}" '.log')
-      le follow --name="${log_name}" "${log_path}" 2>/dev/null 1>&2
+      le follow --name="${log_name}" "${log_path}"
     fi
   done
 
   service logentries restart
 fi
-
-echo "installing aws cfn-tools"
-easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
-cfn-init --region="${AWS_REGION}" --stack="${AWS_STACK_NAME}" --resource="${AWS_INSTANCE_LOGICAL_NAME}" || die 'Failed to run cfn-init'
-

data/priv/roles/NAT.sh

@@ -21,8 +21,4 @@ sysctl -q -w net.ipv4.ip_forward=1 net.ipv4.conf.eth0.send_redirects=0 && (
 sysctl net.ipv4.ip_forward net.ipv4.conf.eth0.send_redirects
 iptables -n -t nat -L POSTROUTING
 
-# reset PrivateRouteThroughBastionBox to point to this instance
-AWS_INSTANCE_ID=$(curl -sL http://169.254.169.254/latest/meta-data/instance-id)
-aws --region="${AWS_REGION}" ec2 replace-route --route-table-id "${NAT_PRIVATE_ROUTE_TABLE}" --destination-cidr-block "0.0.0.0/0" --instance-id "${AWS_INSTANCE_ID}"
-
 echo "Configuration of NAT complete."

data/priv/stack.json.erb

@@ -3,8 +3,9 @@
   "Description": <%= j stack.description %>,
 
   "Parameters" : {
+    "BaseAMIVar": {"Type": "String", "Default": "ami-ddaed3ed"},
     "InstanceAMIVar": {"Type": "String", "Default": "ami-ddaed3ed"},
-    "DockerLibrarySnapshotVar": {"Type": "String", "Default": "snap-d37e4721"},
+
     "IAMKeypairNameVar": {"Type": "String", "MinLength": "5"},
 
     <% stack.roles.each do |role_name| %>