broadside 1.0.3 → 1.1.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0536213915fa9c027fe33ff15b4e9bec02a07b15
-  data.tar.gz: 9ae58ab78407da3477ebb1f2eee07264b7b7477a
+  metadata.gz: dd26f89e64ffb9a89fbeba5e5b51a72b4bd8588b
+  data.tar.gz: 3d1d2f828a45cb8968851e37ef5dbd9ed5d5e01d
 SHA512:
-  metadata.gz: 374b5c4d34b93cc0c357d39dce660c4d617895e5183bfb5bb30e17977be45dbe0b31b5dee0c04046f8eb27faf5bba984466c9b8b6380b4c8f0188c2ef8ae7838
-  data.tar.gz: 2a6fa73a8ce47272c3613d35eccfb04fe55677b294aebeeab99dc95dfe2c4cd03d328f416946db290b7617982e6b1435d911e1c287656e5706b14b4ffcb7a59a
+  metadata.gz: 6b4ddb6d9f87222b23fa9f626d96d0eb3ffd4f48e5b351bb71ea154e18f43f79eb80910629b0d0400b222b00b820f2beb802ea7baf1bcc49a4ce46fa5a1450fc
+  data.tar.gz: 993493358dca3151d56f3a8e7b200a804b7c16297dea2c1c0bb9e722d0c9a68bf3d8a65d659caae37d00ebeeed32ef75ce32a04b2ae01a51f28f023f04abe541

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+# 1.1.0
+- [#16](https://github.com/lumoslabs/broadside/pull/16): Add bootstrap command; add specs
+
 # 1.0.3
 - [#12](https://github.com/lumoslabs/broadside/issues/12): Fix isssue with not being to use ssh, bash, logtail commands without specifying instance index
 

data/README.md

@@ -29,16 +29,36 @@ Broadside.configure do |config|
       scale: 15,
       command: ['bundle', 'exec', 'rake', 'resque:work'],
       env_file: '../.env.production'
-    }
+    },
     staging_web: {
       scale: 1,
       command: ['bundle', 'exec', 'puma'],
       env_file: '../.env.staging'
-    }
+    },
     staging_worker: {
       scale: 1,
       command: ['bundle', 'exec', 'rake', 'resque:work'],
       env_file: '../.env.staging'
+    },
+    # Example with a task_definition and service configuration which you use to bootstrap a service and
+    # initial task definition
+    game_save_as_json_blob_stream: {
+      scale: 1,
+      command: ['java', '-cp', '*:.', 'path.to.MyClass'],
+      env_file: '.env.production',
+      service_config: {
+        deployment_configuration: {
+          minimum_healthy_percent: 0.5,
+        }
+      },
+      task_definition_config: {
+        container_definitions: [
+          {
+            cpu: 1,
+            memory: 2000,
+          }
+        ]
+      }
     }
   }
 end

data/bin/broadside

@@ -42,6 +42,11 @@ accept Fixnum do |val|
   val.to_i
 end
 
+desc 'Bootstrap your service and task definition.'
+command :bootstrap do |b|
+  add_shared_deploy_configs(b)
+end
+
 desc 'Deploy your application.'
 command :deploy do |d|
   d.desc 'Deploys without running migrations'

data/broadside.gemspec

@@ -17,10 +17,12 @@ Gem::Specification.new do |spec|
   spec.executables   = ['broadside']
   spec.require_paths = ["lib"]
 
+  spec.add_dependency 'activesupport', '>= 3', '< 5'
   spec.add_dependency 'aws-sdk', '~> 2.3'
-  spec.add_dependency 'rainbow', '~> 2.1'
-  spec.add_dependency 'gli', '~> 2.13'
   spec.add_dependency 'dotenv', '>= 0.9.0'
+  spec.add_dependency 'gli', '~> 2.13'
+  spec.add_dependency 'rainbow', '~> 2.1'
+
   spec.add_development_dependency 'rspec'
   spec.add_development_dependency 'bundler', '~> 1.9'
   spec.add_development_dependency 'fakefs'

data/lib/broadside.rb

@@ -4,10 +4,10 @@ require 'broadside/configuration'
 require 'broadside/configuration/struct'
 require 'broadside/configuration/aws'
 require 'broadside/configuration/base'
-require 'broadside/configuration/deploy'
-require 'broadside/configuration/ecs'
+require 'broadside/configuration/deploy_config'
+require 'broadside/configuration/ecs_config'
 require 'broadside/deploy'
-require 'broadside/deploy/ecs'
+require 'broadside/deploy/ecs_deploy'
 require 'broadside/version'
 
 module Broadside

data/lib/broadside/configuration/{deploy.rb → deploy_config.rb}

@@ -22,14 +22,18 @@ module Broadside
         :env_vars,
         :command,
         :instance,
-        :predeploy_commands
+        :predeploy_commands,
+        :service_config,
+        :task_definition_config
       )
 
       TARGET_ATTRIBUTE_VALIDATIONS = {
         scale: ->(target_attribute) { validate_types([Fixnum], target_attribute) },
         env_file: ->(target_attribute) { validate_types([String], target_attribute) },
         command: ->(target_attribute) { validate_types([Array, NilClass], target_attribute) },
-        predeploy_commands: ->(target_attribute) { validate_predeploy(target_attribute) }
+        predeploy_commands: ->(target_attribute) { validate_predeploy(target_attribute) },
+        service_config: ->(target_attribute) { validate_types([Hash, NilClass], target_attribute) },
+        task_definition_config: ->(target_attribute) { validate_types([Hash, NilClass], target_attribute) }
       }
 
       def initialize
@@ -45,6 +49,8 @@ module Broadside
         @command = nil
         @predeploy_commands = DEFAULT_PREDEPLOY_COMMANDS
         @instance = 0
+        @service_config = nil
+        @task_definition_config = nil
       end
 
       # Validates format of deploy targets
@@ -86,6 +92,8 @@ module Broadside
         @scale ||= @targets[@target][:scale]
         @command = @targets[@target][:command]
         @predeploy_commands = @targets[@target][:predeploy_commands] if @targets[@target][:predeploy_commands]
+        @service_config = @targets[@target][:service_config]
+        @task_definition_config = @targets[@target][:task_definition_config]
       end
 
       private
@@ -100,14 +108,12 @@ module Broadside
 
       def self.validate_predeploy(target_attribute)
         return nil if target_attribute.nil?
-
         return 'predeploy_commands must be an array' unless target_attribute.is_a?(Array)
 
-        target_attribute.each do |command|
-          return "predeploy_command '#{command}' must be an array" unless command.is_a?(Array)
+        messages = target_attribute.reject { |cmd| cmd.is_a?(Array) }.map do |command|
+          "predeploy_command '#{command}' must be an array" unless command.is_a?(Array)
         end
-
-        nil
+        messages.empty? ? nil : messages.join(', ')
       end
     end
   end

data/lib/broadside/deploy.rb

@@ -92,7 +92,7 @@ module Broadside
     end
 
     def gen_ssh_cmd(ip, options = { tty: false })
-      opts = @deploy_config.ssh
+      opts = @deploy_config.ssh || {}
       cmd = 'ssh -o StrictHostKeyChecking=no'
       cmd << ' -t -t' if options[:tty]
       cmd << " -i #{opts[:keyfile]}" if opts[:keyfile]

data/lib/broadside/deploy/{ecs.rb → ecs_deploy.rb}

@@ -1,3 +1,4 @@
+require 'active_support/core_ext/hash'
 require 'aws-sdk'
 require 'open3'
 require 'pp'
@@ -6,15 +7,23 @@ require 'shellwords'
 
 module Broadside
   class EcsDeploy < Deploy
+
+    DEFAULT_DESIRED_COUNT = 0
+    DEFAULT_CONTAINER_DEFINITION = {
+      cpu: 1,
+      essential: true,
+      memory: 1000
+    }
+
     def initialize(opts)
       super(opts)
       config.ecs.verify(:cluster, :poll_frequency)
-      check_service_existence
     end
 
     def deploy
       super do
-        update_task
+        exception "Service #{family} does not exist!" unless service_exists?
+        update_task_revision
 
         begin
           update_service
@@ -22,17 +31,34 @@ module Broadside
           error 'Caught interrupt signal, rolling back...'
           rollback(1)
           error 'Deployment did not finish successfully.'
-          abort
+          raise
         rescue StandardError => e
           error e.inspect, "\n", e.backtrace.join("\n")
           error 'Deploy failed! Rolling back...'
           rollback(1)
           error 'Deployment did not finish successfully.'
-          abort
+          raise
         end
       end
     end
 
+    def bootstrap
+      unless get_latest_task_def_id
+        # Right now this creates a useless first revision and then update_task_revision will create a 2nd one
+        raise ArgumentError, "No first task definition and cannot create one" unless @deploy_config.task_definition_config
+
+        info "Creating an initial task definition for '#{family}' from the config..."
+        create_task_definition(family, @deploy_config.task_definition_config)
+      end
+
+      unless service_exists?
+        raise ArgumentError, "Service doesn't exist and cannot be created" unless @deploy_config.service_config
+
+        info "Service '#{family}' doesn't exist, creating..."
+        create_service(family, @deploy_config.service_config)
+      end
+    end
+
     def rollback(count = @deploy_config.rollback)
       super do
         begin
@@ -53,7 +79,7 @@ module Broadside
 
     def run
       super do
-        update_task
+        update_task_revision
 
         begin
           run_command(@deploy_config.command)
@@ -66,7 +92,7 @@ module Broadside
     # runs before deploy commands using the latest task definition
     def run_predeploy
       super do
-        update_task
+        update_task_revision
 
         begin
           @deploy_config.predeploy_commands.each do |command|
@@ -94,7 +120,7 @@ module Broadside
 
     def logtail
       super do
-        ip = get_running_instance_ip_at_index!(@deploy_config.instance)
+        ip = get_running_instance_ips.fetch(@deploy_config.instance)
         debug "Tailing logs for running container at ip #{ip}..."
         search_pattern = Shellwords.shellescape(family)
         cmd = "docker logs -f --tail=10 `docker ps -n 1 --quiet --filter name=#{search_pattern}`"
@@ -105,7 +131,7 @@ module Broadside
 
     def ssh
       super do
-        ip = get_running_instance_ip_at_index!(@deploy_config.instance)
+        ip = get_running_instance_ips.fetch(@deploy_config.instance)
         debug "Establishing an SSH connection to ip #{ip}..."
         exec gen_ssh_cmd(ip)
       end
@@ -113,7 +139,7 @@ module Broadside
 
     def bash
       super do
-        ip = get_running_instance_ip_at_index!(@deploy_config.instance)
+        ip = get_running_instance_ips.fetch(@deploy_config.instance)
         debug "Running bash for running container at ip #{ip}..."
         search_pattern = Shellwords.shellescape(family)
         cmd = "docker exec -i -t `docker ps -n 1 --quiet --filter name=#{search_pattern}` bash"
@@ -124,49 +150,65 @@ module Broadside
 
     private
 
-    def get_running_instance_ip_at_index(index)
-      ips = get_running_instance_ips
-      ips.size > index ? ips[index] : false
-    end
-
-    def get_running_instance_ip_at_index!(index)
-      ip = get_running_instance_ip_at_index(index)
-      return ip if ip
-      exception 'Instance index greater than available instance count.'
-    end
-
     # removes latest n task definitions
     def deregister_tasks(count)
-      get_task_def_ids.last(count).each do |td_id|
+      get_task_definition_arns.last(count).each do |td_id|
         ecs_client.deregister_task_definition({task_definition: td_id})
         debug "Deregistered #{td_id}"
       end
     end
 
     # creates a new task revision using current directory's env vars and provided tag
-    def update_task
-      debug "Creating a new task definition..."
+    def update_task_revision
       new_task_def = create_new_task_revision
 
-      container_def = new_task_def[:container_definitions].select { |c| c[:name] == family }.first
-      container_def[:environment] = @deploy_config.env_vars
-      container_def[:image] = image_tag
-      container_def[:command] = @deploy_config.command
+      new_task_def[:container_definitions].select { |c| c[:name] == family }.first.tap do |container_def|
+        container_def[:environment] = @deploy_config.env_vars
+        container_def[:image] = image_tag
+        container_def[:command] = @deploy_config.command
+      end
 
+      debug "Creating a new task definition..."
       new_task_def_id = ecs_client.register_task_definition(new_task_def).task_definition.task_definition_arn
       debug "Successfully created #{new_task_def_id}"
     end
 
+    def create_service(name, options = {})
+      ecs_client.create_service(
+        {
+          cluster: config.ecs.cluster,
+          desired_count: DEFAULT_DESIRED_COUNT,
+          service_name: name,
+          task_definition: name
+        }.deep_merge(options)
+      )
+    end
+
+    def create_task_definition(name, options = {})
+      # Deep merge doesn't work with arrays, so build the hash and merge later
+      container_definitions = DEFAULT_CONTAINER_DEFINITION.merge(
+        name: name,
+        command: @deploy_config.command,
+        environment: @deploy_config.env_vars,
+        image: image_tag,
+      ).merge(options[:container_definitions].first || {})
+
+      ecs_client.register_task_definition(
+        { family: name }.deep_merge(options).merge(container_definitions: [container_definitions])
+      )
+    end
+
     # reloads the service using the latest task definition
     def update_service
       td = get_latest_task_def_id
       debug "Updating #{family} with scale=#{@deploy_config.scale} using task #{td}..."
-      resp =  ecs_client.update_service({
+      resp = ecs_client.update_service({
         cluster: config.ecs.cluster,
         service: family,
         task_definition: get_latest_task_def_id,
         desired_count: @deploy_config.scale
       })
+
       if resp.successful?
         begin
           ecs_client.wait_until(:services_stable, {cluster: config.ecs.cluster, services: [family]}) do |w|
@@ -208,6 +250,7 @@ module Broadside
         count: 1,
         started_by: "before_deploy:#{command_name}"[0...36]
       })
+
       if resp.successful?
         task_id = resp.tasks[0].task_arn
         debug "Launched #{command_name} task #{task_id}"
@@ -221,7 +264,7 @@ module Broadside
         end
         debug 'Task finished running, getting logs...'
         info "#{command_name} task container logs:\n#{get_container_logs(task_id)}"
-        if(code = get_task_exit_code(task_id)) == 0
+        if (code = get_task_exit_code(task_id)) == 0
           debug "#{command_name} task #{task_id} exited with status code 0"
         else
           exception "#{command_name} task #{task_id} exited with a non-zero status code #{code}!"
@@ -259,7 +302,7 @@ module Broadside
     def get_running_instance_ips(task_ids = nil)
       task_arns = nil
       if task_ids.nil?
-        task_arns = get_tasks
+        task_arns = get_task_arns
         if task_arns.empty?
           exception "No running tasks found for '#{family}' on cluster '#{config.ecs.cluster}' !"
         end
@@ -277,44 +320,32 @@ module Broadside
       ec2_instance_ids = container_instances.map { |ci| ci.ec2_instance_id }
       reservations = ec2_client.describe_instances({instance_ids: ec2_instance_ids}).reservations
       instances = reservations.map { |r| r.instances }.flatten
-      private_ips = instances.map { |i| i.private_ip_address }
-      private_ips
+
+      instances.map { |i| i.private_ip_address }
     end
 
     def get_latest_task_def
       ecs_client.describe_task_definition({task_definition: get_latest_task_def_id}).task_definition.to_h
     end
 
-    def get_tasks
-      # used for pagination if > 100 tasks
-      next_token = nil
-      tasks = []
-      loop do
-        resp = ecs_client.list_tasks({cluster: config.ecs.cluster, family: family, next_token: next_token})
-        next_token = resp.next_token
-        tasks += resp.task_arns
-        break if next_token.nil?
-      end
+    def get_task_arns
+      all_results(:list_tasks, :task_arns, { cluster: config.ecs.cluster, family: family })
+    end
 
-      tasks
+    def get_task_definition_arns
+      all_results(:list_task_definitions, :task_definition_arns, { family_prefix: family })
     end
 
-    def get_task_def_ids
-      # used for pagination if > 100 task_definition ids
-      next_token = nil
-      task_def_ids = []
-      loop do
-        resp = ecs_client.list_task_definitions({family_prefix: family, next_token: next_token})
-        next_token = resp.next_token
-        task_def_ids += resp.task_definition_arns
-        break if next_token.nil?
-      end
+    def list_task_definition_families
+      all_results(:list_task_definition_families, :families)
+    end
 
-      task_def_ids
+    def list_services
+      all_results(:list_services, :service_arns, { cluster: config.ecs.cluster })
     end
 
     def get_latest_task_def_id
-      get_task_def_ids.last
+      get_task_definition_arns.last
     end
 
     def create_new_task_revision
@@ -326,6 +357,11 @@ module Broadside
       task_def
     end
 
+    def service_exists?
+      services = ecs_client.describe_services({ cluster: config.ecs.cluster, services: [family] })
+      services.failures.empty? && !services.services.empty?
+    end
+
     def ecs_client
       @ecs_client ||= Aws::ECS::Client.new({
         region: config.aws.region,
@@ -340,11 +376,16 @@ module Broadside
       })
     end
 
-    def check_service_existence
-      resp = ecs_client.describe_services({cluster: config.ecs.cluster, services: [family]})
-      unless resp.failures.empty?
-        exception "Could not find ECS service '#{family}' in cluster '#{config.ecs.cluster}' !"
+    def all_results(method, key, args = {})
+      page = ecs_client.public_send(method, args)
+      results = page.send(key)
+
+      while page.next_token
+        page = ecs.send(method, args.merge(next_token: page.next_token))
+        results += page.send(key)
       end
+
+      results
     end
   end
 end

data/lib/broadside/version.rb

@@ -1,3 +1,3 @@
 module Broadside
-  VERSION = '1.0.3'
+  VERSION = '1.1.0.pre'
 end

metadata

@@ -1,15 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: broadside
 version: !ruby/object:Gem::Version
-  version: 1.0.3
+  version: 1.1.0.pre
 platform: ruby
 authors:
 - Matthew Leung
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-08-25 00:00:00.000000000 Z
+date: 2016-10-01 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '5'
 - !ruby/object:Gem::Dependency
   name: aws-sdk
   requirement: !ruby/object:Gem::Requirement
@@ -25,19 +45,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.3'
 - !ruby/object:Gem::Dependency
-  name: rainbow
+  name: dotenv
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.9.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: gli
   requirement: !ruby/object:Gem::Requirement
@@ -53,19 +73,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '2.13'
 - !ruby/object:Gem::Dependency
-  name: dotenv
+  name: rainbow
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '2.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ">="
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.9.0
+        version: '2.1'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -131,11 +151,11 @@ files:
 - lib/broadside/configuration.rb
 - lib/broadside/configuration/aws.rb
 - lib/broadside/configuration/base.rb
-- lib/broadside/configuration/deploy.rb
-- lib/broadside/configuration/ecs.rb
+- lib/broadside/configuration/deploy_config.rb
+- lib/broadside/configuration/ecs_config.rb
 - lib/broadside/configuration/struct.rb
 - lib/broadside/deploy.rb
-- lib/broadside/deploy/ecs.rb
+- lib/broadside/deploy/ecs_deploy.rb
 - lib/broadside/error.rb
 - lib/broadside/utils.rb
 - lib/broadside/version.rb
@@ -154,9 +174,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.2.3