ecs-tools 0.0.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 007e703284dfe0d89de72cc5a06ca50e1755259d
+  data.tar.gz: aa11ceabcbd5210e198b264ab7c8a2f4f0c9d8fa
+SHA512:
+  metadata.gz: 5424d08bdae0603d0a92df619ef96eba32b947078c1f13c6fefda9b165044e276f7d27227e4f84eaf824dc53d006e66fa405435d4c8e89eeab6f615f7b1b2c3c
+  data.tar.gz: 8eff845e545b8d20193eec37c0f074ae849d22c75f81851b1ca83b9c8a250e36313b5bcb762da1c6aa3cf296846aeb4cf4aef834a225541437787a4d7f268575

data/bin/ecr-build

@@ -0,0 +1,104 @@
+#!/usr/bin/env ruby
+#  ecr-build: Simple script for building docker images and pushing them to Amazon Elastic Container Service Registry (ECR)
+#  
+#  Usage: ecr-build [options]...
+#  
+#  Arguments:
+#  
+#     -k | --aws-access-key  AWS Access Key ID. May also be set as environment variable AWS_ACCESS_KEY_ID
+#     -s | --aws-secret-key  AWS Secret Access Key. May also be set as environment variable AWS_SECRET_ACCESS_KEY
+#     -r | --region          AWS Region Name. May also be set as environment variable AWS_DEFAULT_REGION
+#  
+#     -f | --file            Location of the Dockerfile (default: Dockerfile)
+#     -r | --repository-url  Specify the ECR repository url (default: url from first repository)
+#     -t | --tag             Tag of the docker image, eg. latest
+#  
+require "open3"
+$stderr.sync = true
+require "ecs_deploy"
+
+def usage
+  exec "grep '^#  ' < '#{__FILE__}' | cut -c4-"
+end
+
+def format_image_tag(tag)
+  STDERR.puts "Warning: Provided image tag is longer than 30 characters. Resulting tag will be cut short." if tag.length > 30
+  tag[0..29].gsub(/[^a-z0-9_.-]/, ".")
+end
+
+def cmd(command)
+  # popen2e merges stdout and stderr
+  Open3.popen2e(command) do |stdin, output, thread|
+    while line = output.gets do
+      puts line
+    end
+    process_status = thread.value
+    unless process_status.success?
+      STDERR.puts "Error: '#{command}' failed with status #{process_status.exitstatus}"
+      exit 1
+    end
+  end
+end
+
+if ARGV.first == "format-image-tag"
+  ARGV.shift
+  puts format_image_tag(ARGV.shift)
+  exit 0
+end
+
+# default options
+aws_access_key = ENV["AWS_ACCESS_KEY_ID"]
+aws_secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+aws_region = ENV["AWS_REGION"] || ENV["AWS_DEFAULT_REGION"]
+dockerfile = "Dockerfile"
+repository = nil
+image_tag = "latest"
+
+# parse arguments
+ARGV.options do |opts|
+  # aws options
+  opts.on("-k", "--aws-access-key=key", String) { |v| aws_access_key = v }
+  opts.on("-s", "--aws-secret-key=secret", String) { |v| aws_secret_key = v }
+  opts.on("-r", "--region=aws-region", String) { |v| aws_region = v }
+  # ecr-build options
+  opts.on("-f", "--file=name", String) { |v| dockerfile = v }
+  opts.on("-r", "--repository-url=url", String) { |v| repository = v }
+  opts.on("-t", "--tag=name", String) { |v| image_tag = v }
+  opts.on_tail("-h", "--help") { usage }
+  opts.parse!
+end
+
+unless File.exist?(dockerfile)
+  STDERR.puts "Dockerfile does not exist at #{`pwd`.strip}/#{dockerfile}"
+  exit 1
+end
+
+unless repository
+  STDERR.puts "Repository url must be defined using -r/--repository-url=url"
+  exit 1
+end
+
+# configure Aws
+Aws.config.update(region: aws_region, credentials: Aws::Credentials.new(aws_access_key, aws_secret_key))
+ecr = Aws::ECR::Client.new
+
+image_tag = format_image_tag(image_tag)
+image_name = repository.sub(/\/.*$/, ":#{image_tag}")
+target = "#{repository}:#{image_tag}"
+
+# build the docker image
+cmd "docker build --file #{dockerfile} --tag #{image_name} ."
+
+# tag the docker image
+cmd "docker tag #{image_name} #{target}"
+
+# login
+authorization_response = ecr.get_authorization_token
+authorization_data = authorization_response.authorization_data.first
+user, token = Base64.decode64(authorization_data.authorization_token).split(":")
+cmd "docker login -u '#{user}' -p '#{token}' -e none #{authorization_data.proxy_endpoint}"
+
+# push the image
+cmd "docker push #{target}"
+puts target
+

data/bin/ecs-console

@@ -0,0 +1,95 @@
+#!/usr/bin/env ruby
+#  ecs-console: Simple script for connecting you to a docker instance on Amazon Elastic Container Service
+#  
+#  Usage: ecs-console [options] <command>
+#  
+#  Arguments:
+#  
+#     -k | --aws-access-key  AWS Access Key ID. May also be set as environment variable AWS_ACCESS_KEY_ID
+#     -s | --aws-secret-key  AWS Secret Access Key. May also be set as environment variable AWS_SECRET_ACCESS_KEY
+#     -r | --region          AWS Region Name. May also be set as environment variable AWS_DEFAULT_REGION
+#     -c | --cluster         Specify the name of the cluster to use
+#     -p | --private-key     Path to the EC2 *.pem private key file (default: '~/.ssh/<cluster>.pem')
+#  
+#     -i | --image           Full repository url to the docker image
+#     -n | --container-name  Specify the name of the container definition of your ecs_deploy.yml
+#     -s | --service         Attach to a running service (alternative to image/container)
+#     -f | --file            Use the following ecs config file (default: config/ecs_deploy.yml)
+#  
+$stderr.sync = true
+require "ecs_deploy"
+require "net/scp"
+require 'byebug'
+
+def usage
+  exec "grep '^#  ' < '#{__FILE__}' | cut -c4-"
+end
+
+def repository_slug
+  `git remote -v`.scan(/origin.+github.com:([^\.]+)\.git/).flatten.first
+end
+
+# default options
+aws_access_key = ENV["AWS_ACCESS_KEY_ID"]
+aws_secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+aws_region = ENV["AWS_REGION"] || ENV["AWS_DEFAULT_REGION"]
+file = branch = image = cluster = container = service = private_key = nil
+
+# parse arguments
+ARGV.options do |opts|
+  opts.on("-b", "--branch=git-branch", String) { |v| branch = v }
+  # aws/ecs options
+  opts.on("-k", "--aws-access-key", String) { |v| aws_access_key = v }
+  opts.on("-s", "--aws-secret-key", String) { |v| aws_secret_key = v }
+  opts.on("-r", "--region=aws-region", String) { |v| aws_region = v }
+  opts.on("-c", "--cluster=name", String) { |v| cluster = v }
+  opts.on("-p", "--private-key=path", String) { |v| private_key = v  }
+  # ecs-console options
+  opts.on("-i", "--image=name", String) { |v| image = v }
+  opts.on("-n", "--container-name=name", String) { |v| container = v }
+  opts.on("-s", "--service=name", String) { |v| service = v }
+  opts.on("-f", "--file=path", String) { |v| file = v }
+
+  opts.on_tail("-h", "--help") { usage }
+  opts.parse!
+end
+
+file ||= "config/ecs_#{branch}.yml"
+command = ARGV.join(" ")
+
+# configure Aws
+Aws.config.update(region: aws_region, credentials: Aws::Credentials.new(aws_access_key, aws_secret_key))
+ec2 = Aws::EC2::Client.new
+ecs = Aws::ECS::Client.new
+unless image
+  ecr = Aws::ECR::Client.new
+  repository_uri = ecr.describe_repositories.repositories.find { |repository| repository.repository_name == repository_slug }.repository_uri
+  image_details = ecr.describe_images(repository_name: repository_slug).image_details
+  image_details = image_details.select { |image_detail| image_detail.image_tags&.first.to_s =~ /^#{branch}/ }
+  last_image_tag = image_details.sort_by(&:image_pushed_at).last.image_tags.first
+  image = "#{repository_uri}:#{last_image_tag}"
+end
+
+config = YAML.load_file(file).deep_symbolize_keys
+container ||= config[:container_definitions].keys.first
+container_definition = config[:container_definitions][container.to_sym]
+cluster ||= config[:cluster]
+cluster_name = cluster.sub(/-ECSCluster.+$/, '')
+private_key ||= "~/.ssh/#{cluster_name}.pem"
+
+container_instance_arns = ecs.list_container_instances(cluster: cluster).container_instance_arns
+instance_id = ecs.describe_container_instances(cluster: cluster, container_instances: container_instance_arns).container_instances.sample.ec2_instance_id
+public_dns_name = ec2.describe_instances(instance_ids: [instance_id]).reservations.first.instances.first.public_dns_name
+
+environment_data = container_definition[:environment].map { |env| "#{env[:name]}=#{env[:value]}" }.join("\n")
+remote_env_file = "/home/ec2-user/.ecs-console.#{Time.now.to_f}.tmp.env"
+Net::SCP.upload!(public_dns_name, "ec2-user", StringIO.new(environment_data), remote_env_file, ssh: {keys: [File.expand_path(private_key)]})
+
+puts "Connecting to ec2-user@#{public_dns_name}, running '#{command}' inside #{image}"
+ssh_command = [
+  "chmod 400 #{remote_env_file}",
+  "\\$(aws ecr get-login --region #{aws_region})",
+  "docker run --rm --env-file=#{remote_env_file} -ti #{image} #{command}",
+  "rm -f #{remote_env_file}"
+].join("; ")
+exec %Q[ssh -ti "#{private_key}" ec2-user@#{public_dns_name} "#{ssh_command}"]

data/bin/ecs-deploy

@@ -0,0 +1,165 @@
+#!/usr/bin/env ruby
+#  ecs-deploy: Simple script for deploying docker images to Amazon Elastic Container Service
+#  
+#  Usage: deploy [options]...
+#  
+#  Arguments:
+#  
+#     -k | --aws-access-key  AWS Access Key ID. May also be set as environment variable AWS_ACCESS_KEY_ID
+#     -s | --aws-secret-key  AWS Secret Access Key. May also be set as environment variable AWS_SECRET_ACCESS_KEY
+#     -r | --region          AWS Region Name. May also be set as environment variable AWS_DEFAULT_REGION
+#  
+#     -i | --image           Full repository url to the docker image
+#     -c | --cluster         Specify the name of the cluster to use
+#     -w | --wait-time       Time given to wait until all services have the new task definition running
+#     -f | --file            ECS deploy configuration file (default: config/ecs_deploy.yml)
+#  
+$stderr.sync = true
+require "ecs_deploy"
+
+def usage
+  exec "grep '^#  ' < '#{__FILE__}' | cut -c4-"
+end
+
+class Aws::ECS::Client
+  def upsert_service(options)
+    send(:update_service, options)
+  rescue Aws::ECS::Errors::ServiceNotFoundException, Aws::ECS::Errors::ServiceNotActiveException
+    options[:service_name] = options.delete(:service)
+    send(:create_service, options)
+  end
+end
+
+# default options
+aws_access_key = ENV["AWS_ACCESS_KEY_ID"]
+aws_secret_key = ENV["AWS_SECRET_ACCESS_KEY"]
+aws_region = ENV["AWS_REGION"] || ENV["AWS_DEFAULT_REGION"]
+image = cluster = nil
+debug = false
+wait_time = 420 # 7 minutes
+config_file = "config/ecs_deploy.yml"
+
+# parse arguments
+ARGV.options do |opts|
+  # aws options
+  opts.on("-k", "--aws-access-key=key", String) { |v| aws_access_key = v }
+  opts.on("-s", "--aws-secret-key=secret", String) { |v| aws_secret_key = v }
+  opts.on("-r", "--region=aws-region", String) { |v| aws_region = v }
+  # ecs-deploy options
+  opts.on("-i", "--image=name", String) { |v| image = v }
+  opts.on("-c", "--cluster=name", String) { |v| cluster = v }
+  opts.on("-w", "--wait-time=seconds", Integer) { |v| wait_time = v }
+  opts.on("-f", "--file=path", String) { |v| config_file = v }
+  # opts.on("-i", "--int=val", Integer)  { |val| integer = val }
+  # opts.on("--list=[x,y,z]", Array)     { |val| list = val }
+  opts.on("--debug") { debug = true }
+  opts.on_tail("-h", "--help") { usage }
+  opts.parse!
+end
+
+# configure Aws
+Aws.config.update(region: aws_region, credentials: Aws::Credentials.new(aws_access_key, aws_secret_key))
+ecs = Aws::ECS::Client.new
+
+config = YAML.load_file(config_file).deep_symbolize_keys
+
+cluster ||= config[:cluster]
+
+# register new task definitions
+task_definitions = {}
+config[:task_definitions].each do |family, task_definition|
+  definition = deep_dup(task_definition)
+  definition[:family] = family
+  definition[:container_definitions].each { |definition| definition[:image] = image } if image
+  response = ecs.register_task_definition(definition)
+  definition[:arn] = response.task_definition.task_definition_arn
+  puts "New task definition: #{definition[:arn]}"
+  task_definitions[family] = definition
+end
+
+def report_run_task_failures(failures)
+  return if failures.empty?
+  failures.each do |failure|
+    STDERR.puts "Error: run task failure '#{failure.reason}'"
+  end
+  exit 1
+end
+
+# handle one off tasks
+config.fetch(:one_off_commands, []).each do |one_off_command|
+  task_definition = task_definitions[one_off_command[:task_family].to_sym]
+  puts "Running '#{one_off_command[:command]}'"
+  response = ecs.run_task(
+    cluster: cluster,
+    task_definition: task_definition[:arn],
+    count: 1,
+    started_by: "ecs-deploy: one_off_commands",
+    overrides: {
+      container_overrides: [
+        {
+          name: task_definition[:container_definitions].first[:name],
+          command: Array(one_off_command[:command])
+        }
+      ]
+    }
+  )
+  # handle potential failures
+  report_run_task_failures(response.failures)
+
+  task_arn = response.tasks.first.task_arn
+  print "Waiting for '#{one_off_command[:command]}' to finish"
+  waiting = 0
+  last_now = Time.now
+  task = nil
+  while waiting <= wait_time do
+    task = ecs.describe_tasks(tasks: [task_arn], cluster: cluster).tasks.first
+    break if task.last_status == "STOPPED"
+    print "."
+    now = Time.now
+    waiting += (now - last_now).to_i
+    last_now = now
+    sleep 5
+  end
+  if waiting > wait_time
+    STDERR.puts "Error: wait time exceeded"
+    exit 1
+  end
+  if task.containers.first.exit_code != 0
+    STDERR.puts "Error: '#{one_off_command[:command]}' finished with a non-zero exit code! Aborting."
+    exit 1
+  end
+  puts " done!"
+end
+
+# handle services
+task_definition_arns = []
+config[:services].each do |service|
+  task_definition = task_definitions[service[:task_family].to_sym]
+  task_definition_arns << task_definition[:arn]
+
+  ecs.upsert_service(
+    cluster: cluster,
+    service: service[:name],
+    desired_count: service[:desired_count],
+    task_definition: task_definition[:arn],
+    deployment_configuration: service[:deployment_configuration]
+  )
+end
+
+print "Waiting #{wait_time}s until all service tasks come up running"
+waiting = 0
+last_now = Time.now
+while waiting <= wait_time do
+  running_tasks = ecs.describe_tasks(cluster: cluster, tasks: ecs.list_tasks(cluster: cluster, desired_status: "RUNNING").task_arns).tasks
+  break if task_definition_arns.all? { |arn| running_tasks.any? { |task| task.task_definition_arn == arn } }
+  print "."
+  waiting += (Time.now - last_now).to_i
+  sleep 5
+end
+if waiting > wait_time
+  STDERR.puts "Error: wait time exceeded"
+  exit 1
+else
+  puts " done!"
+end
+

data/lib/ecs_deploy.rb

@@ -0,0 +1,25 @@
+require "optparse"
+require "yaml"
+require "aws-sdk"
+
+def deep_dup(obj)
+  Marshal.load(Marshal.dump(obj))
+end
+
+class Array
+  def deep_symbolize_keys
+    map do |object|
+      object.respond_to?(:deep_symbolize_keys) ? object.deep_symbolize_keys : object
+    end
+  end
+end
+
+class Hash
+  def deep_symbolize_keys
+    inject({}) do |hash, (key, value)|
+      hash[key.to_sym] = value.respond_to?(:deep_symbolize_keys) ? value.deep_symbolize_keys : value
+      hash
+    end
+  end
+end
+

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: ecs-tools
+version: !ruby/object:Gem::Version
+  version: 0.0.4
+platform: ruby
+authors:
+- Lukas Rieder
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-06-13 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: aws-sdk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.14
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 2.8.0
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.8.14
+- !ruby/object:Gem::Dependency
+  name: net-ssh
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+- !ruby/object:Gem::Dependency
+  name: net-scp
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.2'
+description: Deploy your containerized application into Amazon Elastic Container Service
+email: l.rieder@gmail.com
+executables:
+- ecs-deploy
+- ecs-console
+- ecr-build
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/ecr-build
+- bin/ecs-console
+- bin/ecs-deploy
+- lib/ecs_deploy.rb
+homepage: https://github.com/Overbryd/ecs-tools
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.11
+signing_key: 
+specification_version: 4
+summary: AWS ECS deployment
+test_files: []