stacco 0.1.20 → 0.1.21

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 21af987480fa2bb69db33761063b94915053363c
-  data.tar.gz: 14ea2ffc6c4da1dab971f8f2d7c3416cd5234738
+  metadata.gz: 018a261fc9ce87765f84460e1c207a244caad5b4
+  data.tar.gz: 2332400896bccab5424c9f85c835bbfc7231f07a
 SHA512:
-  metadata.gz: 9dd4a46a9ed4a0928f4f957633a91e440c88b33e74dd07bb7e62b253d074a1348d0292ccac79628592ac8118585d17c9cc363bae47be40bc88561786eab54865
-  data.tar.gz: fd4cf5500e6b94424316d994bfedcd7917e565b9ce287ef8fe6cd5f0e4e647dc98e317b532a3aa0e2b98195996a866820572b04dcc1b266710db92a5b8bc7ac4
+  metadata.gz: 3a226439257a465b86cedafe66654eecf2511d09722d0934054d392a7a5354ea11e440ebfa3648384eccdb7b99210fa09516fbf6881f3bacf6048a07c9af6584
+  data.tar.gz: 6de6de1874a865280b9bce02600c2e39edfc71e5cf39b391a4ef4d0eb0f396c956bc00ad393fa8c68ebf7cb019d1fafbcd5f495a0be0821b628417043d04c586

data/.git/COMMIT_EDITMSG

@@ -1 +1 @@
-Merge Bastion and NAT functions, and move DockerHost to using upstart
+Externalize DockerHost logic

data/.git/logs/HEAD

@@ -1,2 +1,3 @@
 0000000000000000000000000000000000000000 43266d16742cc444c81d82f49923f99546bf6702 Levi Aul <levi@leviaul.com> 1402508203 -0700	clone: from git@github.com:bexio/cloudformation.git
 43266d16742cc444c81d82f49923f99546bf6702 7ae0507874a426eccf1e5cade20bbd06e748026a Levi Aul <levi@leviaul.com> 1402598788 -0700	commit: Merge Bastion and NAT functions, and move DockerHost to using upstart
+7ae0507874a426eccf1e5cade20bbd06e748026a 016915ad73dd873159f7f9c9f386eb833f683854 Levi Aul <levi@leviaul.com> 1402619504 -0700	commit: Externalize DockerHost logic

data/.git/logs/refs/heads/master

@@ -1,2 +1,3 @@
 0000000000000000000000000000000000000000 43266d16742cc444c81d82f49923f99546bf6702 Levi Aul <levi@leviaul.com> 1402508203 -0700	clone: from git@github.com:bexio/cloudformation.git
 43266d16742cc444c81d82f49923f99546bf6702 7ae0507874a426eccf1e5cade20bbd06e748026a Levi Aul <levi@leviaul.com> 1402598788 -0700	commit: Merge Bastion and NAT functions, and move DockerHost to using upstart
+7ae0507874a426eccf1e5cade20bbd06e748026a 016915ad73dd873159f7f9c9f386eb833f683854 Levi Aul <levi@leviaul.com> 1402619504 -0700	commit: Externalize DockerHost logic

data/.git/logs/refs/remotes/origin/master

@@ -1 +1,2 @@
 43266d16742cc444c81d82f49923f99546bf6702 7ae0507874a426eccf1e5cade20bbd06e748026a Levi Aul <levi@leviaul.com> 1402598796 -0700	update by push
+7ae0507874a426eccf1e5cade20bbd06e748026a 016915ad73dd873159f7f9c9f386eb833f683854 Levi Aul <levi@leviaul.com> 1402619519 -0700	update by push

data/.git/objects/01/6915ad73dd873159f7f9c9f386eb833f683854

@@ -0,0 +1,2 @@
+x
��]
+�0�}�)��&�O��v�j0m��"��x_�a�>���F/�$

data/.git/objects/1d/24fb1b18e61fee0a5e5a065984b959eb3e26f7

@@ -0,0 +1 @@
+x
e�Mk�0�wΧxȆ�6u��n��aZ6;J���I���!��V�ly�/Ϗ�.`�ona��^����֕E�Q�V�'v��L�ِ�a4J�e2�|0IJ��aϥ���RC*Y~��^�Q-9����\gu����>�iu�A������w�E)��0�d�gC�B�I!�ݭ�yi�rBq 𮇧��s��_7��?�&��1�:V��ܝ:q6Y��Y����wz�]��)*iwSGiő��v�u�8��p
f~3��.�6C��6��Wt��N��

data/.git/refs/heads/master

@@ -1 +1 @@
-7ae0507874a426eccf1e5cade20bbd06e748026a
+016915ad73dd873159f7f9c9f386eb833f683854

data/.git/refs/remotes/origin/master

@@ -1 +1 @@
-7ae0507874a426eccf1e5cade20bbd06e748026a
+016915ad73dd873159f7f9c9f386eb833f683854

data/.git/refs/tags/0.1.21

@@ -0,0 +1 @@
+016915ad73dd873159f7f9c9f386eb833f683854

data/priv/layers/admin-api.json

@@ -77,7 +77,7 @@
 
   "UpdatePolicy": {
     "AutoScalingRollingUpdate": {
-      "MinInstancesInService": "0",
+      "MinInstancesInService": "1",
       "MaxBatchSize": "1",
       "PauseTime": "PT0S"
     }
@@ -89,7 +89,7 @@
     "LaunchConfigurationName": {"Ref": "BackendLaunchConfiguration"},
 
     "MinSize": "1",
-    "MaxSize": "1",
+    "MaxSize": "2",
 
     "HealthCheckType": "ELB",
     "HealthCheckGracePeriod": "1600",
@@ -146,7 +146,6 @@
       "export AWS_INSTANCE_WAIT_HANDLE='", {"Ref": "AdminAPIScalingGroupReadyWaitHandle"}, "'\n",
       {"Ref": "UserDataEnvironmentVar"}, "\n",
       {"Ref": "CommonRoleScriptVar"}, "\n",
-      {"Ref": "DockerHostRoleScriptVar"}, "\n",
       {"Ref": "BackendRoleScriptVar"}, "\n"
     ]]}}
   }

data/priv/layers/client-api.json

@@ -83,7 +83,7 @@
 
   "UpdatePolicy": {
     "AutoScalingRollingUpdate": {
-      "MinInstancesInService": "0",
+      "MinInstancesInService": "1",
       "MaxBatchSize": "1",
       "PauseTime": "PT0S"
     }
@@ -95,7 +95,7 @@
     "LaunchConfigurationName": {"Ref": "FrontendLaunchConfiguration"},
 
     "MinSize": "1",
-    "MaxSize": "1",
+    "MaxSize": "2",
 
     "HealthCheckType": "ELB",
     "HealthCheckGracePeriod": "1600",
@@ -147,7 +147,6 @@
       "export BEXNG_ADMIN_API_HOST='", {"Fn::FindInMap": ["StackZoneRecords", "AdminAPI", "DNSName" ]}, "'\n",
       {"Ref": "UserDataEnvironmentVar"}, "\n",
       {"Ref": "CommonRoleScriptVar"}, "\n",
-      {"Ref": "DockerHostRoleScriptVar"}, "\n",
       {"Ref": "FrontendRoleScriptVar"}, "\n"
     ]]}}
   }

data/priv/layers/task-generate-base-image.json

@@ -0,0 +1,68 @@
+{
+
+"Resources": {
+
+"BaseImageGeneratorSecurityGroup" : {
+  "Type" : "AWS::EC2::SecurityGroup",
+  "Properties" : {
+    "GroupDescription" : "Allow the application instances to access the NAT device",
+    "VpcId" : { "Ref" : "VPC" },
+
+    "SecurityGroupIngress": [
+      {"IpProtocol": "tcp", "FromPort": "22", "ToPort": "22", "CidrIp": "0.0.0.0/0"}
+    ],
+    "SecurityGroupEgress": [
+      {"IpProtocol": "-1", "CidrIp": "0.0.0.0/0"}
+    ]
+  }
+},
+
+"BaseImageGeneratorReadyWaitHandle": {"Type": "AWS::CloudFormation::WaitConditionHandle", "Properties": {}},
+
+"BaseImageGeneratorReady": {"Type": "AWS::CloudFormation::WaitCondition", "DependsOn": ["BaseImageGenerator"], "Properties": {
+  "Handle": {"Ref": "BaseImageGeneratorReadyWaitHandle"},
+  "Count": "1",
+  "Timeout": "1200"
+}},
+
+"BaseImageGenerator" : {
+  "Type" : "AWS::EC2::Instance",
+  "Metadata": {
+    "AWS::CloudFormation::Init": {}
+  },
+  "Properties" : {
+    "InstanceType": "m3.large",
+    "ImageId": {"Ref": "InstanceAMIVar"},
+    "KeyName": {"Ref": "IAMKeypairNameVar"},
+
+    "NetworkInterfaces": [{
+      "DeviceIndex": "0",
+      "AssociatePublicIpAddress": "true",
+      "DeleteOnTermination": "true",
+      "SubnetId": {"Ref": "PublicSubnet"},
+      "GroupSet" : [{"Ref" : "BaseImageGeneratorSecurityGroup"}]
+    }],
+
+    "BlockDeviceMappings": [
+      {"DeviceName": "/dev/xvdc", "Ebs": {
+        "VolumeSize": "5"
+      }}
+    ],
+
+    "UserData": {"Fn::Base64": {"Fn::Join": ["", [
+      "#!/bin/bash\n",
+      "export AWS_REGION='", {"Ref": "AWS::Region"}, "'\n",
+      "export AWS_STACK_NAME='", {"Ref": "AWS::StackName"}, "'\n",
+      "export AWS_INSTANCE_LOGICAL_NAME='BaseImageGenerator'\n",
+      "export AWS_INSTANCE_WAIT_HANDLE='", {"Ref": "BaseImageGeneratorReadyWaitHandle"}, "'\n",
+      {"Ref": "UserDataEnvironmentVar"}, "\n",
+      {"Ref": "CommonRoleScriptVar"}, "\n",
+      {"Ref": "DockerHostRoleScriptVar"}, "\n",
+      {"Ref": "BaseImageGeneratorRoleScriptVar"}, "\n"
+    ]]}}
+  }
+}
+
+}
+
+}

data/priv/roles/Backend.sh

@@ -1,3 +1,8 @@
+# DockerHost tools
+run-gist "tsutsu/72c4ac40591f96ad379f"
+
+mkdir -p /var/lib/docker && mount /dev/xvdc /var/lib/docker
+mkdir -p /volumes && mount /dev/xvdd /volumes
 
 # create bexng file-based config
 mkdir -p /etc/bexng
@@ -21,28 +26,27 @@ cat >"/etc/bexng/secrets.exs" <<EOF
 EOF
 
 
-define_container "quay.io/bexio/postgresql" \
+docker-bootstrap
+
+docker-define-service "quay.io/bexio/postgresql" \
   --volume "/volumes/postgresql:/var/lib/postgresql"
 
-define_container "quay.io/bexio/bitcoind" \
+docker-define-service "quay.io/bexio/bitcoind" \
   --volume "/volumes/bitcoind:/var/lib/bitcoin"
 
-define_container "quay.io/bexio/bexng" \
+docker-define-service "quay.io/bexio/bexng" \
   --dependency "bitcoind" \
   --dependency "postgresql" \
   --volume "/etc/bexng:/target" \
   --publish="51607:51607" \
   -e "BEXNG_TARGET_SYSTEM=${DOMAIN}"
 
-define_container "quay.io/bexio/bexng_frontend" \
+docker-define-service "quay.io/bexio/bexng_frontend" \
   --dependency "bexng" \
   --publish="80:8080"
 
 
-echo "setting up docker:"
-create_docker_library
-
-echo "starting container services:"
+echo "starting docker-container services..."
 start postgresql
 start bitcoind
 start bexng

data/priv/roles/Common.sh

@@ -2,7 +2,6 @@ _exit_handler_reason=""
 _exit_handler(){
   exit_code="$?"
 
-
   if [ "${exit_code}" -eq 0 ]; then
     echo "Stacco cloud-init handler finished"
     cfn-signal -s true "${AWS_INSTANCE_WAIT_HANDLE}"
@@ -22,23 +21,43 @@ die(){
 
 trap _exit_handler EXIT
 set -e
-
-echo "running as $(whoami) in ${PWD}"
 export HOME=/root
 cd "$HOME"
 
 
-echo "ensuring internet connectivity..."
-while true; do
-  nat_status=$(curl -s -o /dev/null -I -w "%{http_code}" -m 2 http://bex-status.s3.amazonaws.com/status.json)
-  if [ "${nat_status}" = "200" ]; then
-    break
-  fi
-  sleep 2
-done
+cat >/usr/local/bin/run-gist <<EOF
+  gist_id="\$1"; shift
+
+  gist_download_url="https://gist.github.com/\${gist_id}/download"
+  extract_dir=\$(mktemp -d /tmp/stacco.XXXXXXXX)
+
+  echo "unpacking '\${gist_download_url}'..."
+
+  pushd "\${extract_dir}" >/dev/null
+    curl -sL "\${gist_download_url}" | tar -x -z --strip=1
+    chmod a+x ./run
+    ./run
+  popd >/dev/null
+
+  rm -rf "\${extract_dir}"
+EOF
+chmod a+x /usr/local/bin/run-gist
 
+
+echo "ensuring internet connectivity..."
+curl -I --silent --fail --show-error --retry 100 http://bex-status.s3.amazonaws.com/status.json
 echo "found an internet connection."
 
+echo "configuring hostname from aws metadata service"
+private_hostname=$(curl http://169.254.169.254/latest/meta-data/hostname)
+echo "${private_hostname}" > /etc/hostname
+hostname -b -F /etc/hostname
+short_hostname=$(hostname -s)
+sed -i '/127\.0\.0\.1/d' /etc/hosts
+cat >>/etc/hosts <<EOF
+127.0.0.1 localhost ${short_hostname} ${private_hostname}
+EOF
+
 echo "disablng kernel and initramfs updates"
 echo $(dpkg -l "*$(uname -r)*" | grep image | awk '{print $2}') hold | dpkg --set-selections
 sed -i 's/=yes/=no/g' /etc/initramfs-tools/update-initramfs.conf
@@ -52,23 +71,13 @@ echo "installing aws cfn-tools"
 easy_install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-latest.tar.gz
 cfn-init --region="${AWS_REGION}" --stack="${AWS_STACK_NAME}" --resource="${AWS_INSTANCE_LOGICAL_NAME}" || die 'Failed to run cfn-init'
 
-echo "configuring hostname from aws metadata service"
-private_hostname=$(curl http://169.254.169.254/latest/meta-data/hostname)
-echo "${private_hostname}" > /etc/hostname
-
-hostname -b -F /etc/hostname
-short_hostname=$(hostname -s)
-
-sed -i '/127\.0\.0\.1/d' /etc/hosts
-cat >>/etc/hosts <<EOF
-127.0.0.1 localhost ${short_hostname} ${private_hostname}
-EOF
+for dev in /dev/xvd* /dev/ephemeral*; do
+  if [ -n "$(mount | awk '{print $1;}' | grep "^${dev}$")" ]; then
+    next
+  fi
 
-if [ -b "/dev/xvdd" ]; then
-  echo "formatting and mounting /volumes storage"
-  mkfs.btrfs -L datavols /dev/xvdd
-  mkdir -p /volumes
-  mount /dev/xvdd /volumes
-  btrfs filesystem resize max /volumes
+  mkfs.btrfs "${dev}" 2>/dev/null || :
+  mount "${dev}" /mnt
+    btrfs filesystem resize max /mnt 2>/dev/null || :
+  umount /mnt
 fi
-

data/priv/roles/Frontend.sh

@@ -1,7 +1,14 @@
-define_container "quay.io/bexio/bexng_frontend" \
+# DockerHost tools
+run-gist "tsutsu/72c4ac40591f96ad379f"
+
+mkdir -p /var/lib/docker && mount /dev/xvdc /var/lib/docker
+mkdir -p /volumes && mount /dev/xvdd /volumes
+
+docker-bootstrap
+
+docker-define-service "quay.io/bexio/bexng_frontend" \
   -e "BEXNG_PORT_51607_TCP=tcp://${BEXNG_ADMIN_API_HOST}:51607" \
   --publish="80:8080"
 
-create_docker_library
-
+echo "starting docker-container services..."
 start bexng_frontend

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: stacco
 version: !ruby/object:Gem::Version
-  version: 0.1.20
+  version: 0.1.21
 platform: ruby
 authors:
 - Levi Aul
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-06-12 00:00:00.000000000 Z
+date: 2014-06-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk
@@ -78,17 +78,28 @@ files:
 - ./.git/logs/refs/heads/master
 - ./.git/logs/refs/remotes/origin/HEAD
 - ./.git/logs/refs/remotes/origin/master
+- ./.git/objects/01/6915ad73dd873159f7f9c9f386eb833f683854
 - ./.git/objects/01/ddd978bb9fbf18a379fb418080b2fc4d5102b6
+- ./.git/objects/03/59b4f788063f520915a41d3d73a0d4631b9dfc
+- ./.git/objects/04/09e63d7e887019239ac04a2ebf184d2bf1abd5
 - ./.git/objects/07/1c3c0ff3bbb5d2dee36124869ac978339aa7d8
+- ./.git/objects/08/d390fad8629347b01e81e36389b6445bb7be6e
 - ./.git/objects/18/ec0e701dcbdb487c91265f32d6d0e12a1cce42
+- ./.git/objects/1d/24fb1b18e61fee0a5e5a065984b959eb3e26f7
 - ./.git/objects/1d/f93d4a34d891a00b6b11d13e8e9c49e2e918d7
 - ./.git/objects/36/ba5be11b41d1823658d7ce80027a97ebad5b49
+- ./.git/objects/3c/a160491652dbac4b486a67368c3d5571a02ee7
 - ./.git/objects/3d/f99825f6039015db4c43abf789d2f1e54649f4
+- ./.git/objects/42/a30010d7fb35e5946ac0858140f8286748a49a
+- ./.git/objects/4a/ded72b0a091965c9cb556c90400e8f116b4a83
 - ./.git/objects/4b/7e51ebdd2d7ae3ef0b0a27d081ce1579f5a073
 - ./.git/objects/53/2b0166a9dda8275dcfcba389a1d782269a127f
+- ./.git/objects/60/cffb71344e0595790bc86f9044b635eb584cd5
+- ./.git/objects/67/bdace601575b64eb0969c3b291da318069bad0
 - ./.git/objects/69/2be5effe74e8c9f60c3a949d383324ac075248
 - ./.git/objects/7a/e0507874a426eccf1e5cade20bbd06e748026a
 - ./.git/objects/7e/d9e61793490a93850a3974ab2ab01bc35f4e15
+- ./.git/objects/9a/7a7d20ee398751b7f7c42b93d0db56d5dbe375
 - ./.git/objects/b8/cd9a56aeab69e861ec8b3449b0910cb9d41d6e
 - ./.git/objects/c7/9379c6c2ef65dbb2e04e183e00803154cb35ab
 - ./.git/objects/e1/9b48d2433a281d1a3d1d37877b0c969277a57e
@@ -100,6 +111,7 @@ files:
 - ./.git/refs/remotes/origin/HEAD
 - ./.git/refs/remotes/origin/master
 - ./.git/refs/tags/0.1.20
+- ./.git/refs/tags/0.1.21
 - ./.gitignore
 - ./bin/stacco
 - ./Dockerfile
@@ -118,10 +130,10 @@ files:
 - ./priv/layers/client-api.json
 - ./priv/layers/docker-librarian.json
 - ./priv/layers/static.json
+- ./priv/layers/task-generate-base-image.json
 - ./priv/layers/vpc.json
 - ./priv/roles/Backend.sh
 - ./priv/roles/Common.sh
-- ./priv/roles/DockerHost.sh
 - ./priv/roles/Frontend.sh
 - ./priv/roles/NAT.sh
 - ./priv/roles/VPN.sh

data/priv/roles/DockerHost.sh

@@ -1,132 +0,0 @@
-
-create_docker_library(){
-
-echo "mounting docker library..."
-docker_volume="/dev/xvdc"
-mkfs.btrfs -L dockergraph "${docker_volume}" 2>/dev/null || :
-mkdir -p /var/lib/docker
-mount -t btrfs -o "rw,noatime,space_cache" "${docker_volume}" /var/lib/docker
-btrfs filesystem resize max /var/lib/docker
-
-echo "installing docker..."
-echo 'deb http://get.docker.io/ubuntu docker main' > /etc/apt/sources.list.d/docker.list
-apt-key adv --keyserver 'hkp://keyserver.ubuntu.com:80' --recv-keys '36A1D7869245C8950F966E92D8576A8BA88D21E9'
-apt-get update
-apt-get install -qy lxc-docker
-gpasswd -a ubuntu docker
-
-echo "logging into ${DOCKER_REGISTRY_URL}..."
-cat >/root/.dockercfg <<EOF
-{"${DOCKER_REGISTRY_URL}": {
-  "auth": "${DOCKER_REGISTRY_AUTH}",
-  "email": "${DOCKER_REGISTRY_EMAIL}"
-}}
-EOF
-
-}
-
-
-define_container(){
-
-echo "defining container service '$1'..."
-
-image_repo=$1
-shift
-
-image_name=$(echo "${image_repo}" | tr '/' ' ' | awk '{print $NF;}')
-data_container_name="${image_name}-data"
-
-image_tag="latest"
-dependencies=""
-volumes=""
-published_ports=""
-misc_args=""
-
-while [ "$#" -gt 0 ]; do
-  case "$1" in
-  --tag)
-    shift; tag_name="$1"; shift
-    image_tag="${tag_name}"
-    ;;
-  --dependency)
-    shift; dep_name="$1"; shift
-    dependencies="${dependencies} ${dep_name}"
-    ;;
-  --volume)
-    shift; volume_arg="$1"; shift
-    volumes="${volumes} ${volume_arg}"
-    ;;
-  --publish-all-ports)
-    shift
-    publishing_all_ports=1
-    ;;
-  *)
-    misc_arg="$1"; shift
-    misc_args="${misc_args} ${misc_arg}"
-    ;;
-  esac
-done
-
-stopping_stanza="stopping docker"
-link_args=""
-for dep_name in ${dependencies}; do
-  stopping_stanza="${stopping_stanza} or stopping ${dep_name}"
-  link_args="${link_args} --link=\"${dep_name}:${dep_name}\""
-done
-
-
-data_container_volume_args=""
-for volume_mapping in ${volumes}; do
-  data_container_volume_args="${data_container_volume_args} --volume=\"${volume_mapping}\""
-done
-
-cat >"/etc/init/${image_name}.conf" <<EOF
-stop on (${stopping_stanza})
-
-console output
-
-env HOME=/root
-
-respawn
-respawn limit 5 20
-
-pre-start script
-  . /etc/environment.local
-
-  echo "creating data container (${data_container_name}) if it doesn't exist..."
-  docker run --name="${data_container_name}" ${data_container_volume_args} "tianon/true:latest" 2>/dev/null || :
-
-  echo "updating image (${image_repo}:${image_tag})..."
-  docker pull "${image_repo}:${image_tag}"
-end script
-
-post-stop script
-  docker rm "${image_name}" || :
-end script
-
-script
-  . /etc/environment.local
-
-  echo "starting container: ${image_name} (from ${image_repo}:${image_tag})..."
-
-  if [ -n "${publishing_all_ports}" ]; then
-    published_port_args=""
-    published_ports="\$(docker inspect --format='{{.Config.ExposedPorts}}' "${image_repo}:${image_tag}" | sed -e 's/[^0-9]/ /g')"
-    for port in \${published_ports}; do
-      published_port_args="\${published_port_args} --publish=\"\${port}:\${port}\""
-    done
-  fi
-
-  exec docker run --rm -i -t --sig-proxy=true \
-    --name="${image_name}" \
-    --volumes-from="${data_container_name}" \
-    \${published_port_args} \
-    ${link_args} \
-    ${misc_args} \
-    "${image_repo}:${image_tag}"
-end script
-EOF
-
-echo "defined."
-
-}