shipitron 1.2.0 → 1.4.0

data/docker-compose.yml

@@ -0,0 +1,54 @@
+version: '3.8'
+services:
+  shipitron:
+    build: .
+    image: outstand/shipitron:dev
+    environment:
+      FIXUID:
+      FIXGID:
+    volumes:
+      - bundler-data:/usr/local/bundle
+      - ~/dev/app:/app # Set this to the application to be shipitron'd
+      - shipitron-home:/home/shipitron
+      - ~/.config/shipitron:/home/shipitron/.config/shipitron
+      - .:/shipitron
+
+  specs:
+    build: .
+    image: outstand/shipitron:dev
+    command: rspec
+    environment:
+      FOG_LOCAL: 'true'
+      FIXUID:
+      FIXGID:
+      BUILDKITE:
+      BUILDKITE_BUILD_URL:
+      BUILDKITE_JOB_ID:
+      BUILDKITE_AGENT_ACCESS_TOKEN:
+    working_dir: /shipitron
+    volumes:
+      - bundler-data:/usr/local/bundle
+      - fog:/fog
+      - shipitron-home:/home/shipitron
+      - .:/shipitron
+
+  release_gem:
+    image: outstand/shipitron:dev
+    command: rake release
+    working_dir: /shipitron
+    environment:
+      FIXUID:
+      FIXGID:
+    volumes:
+      - bundler-data:/usr/local/bundle
+      - shipitron-home:/home/shipitron
+      - .:/shipitron
+      - ~/.dotfiles/gitconfig:/root/.gitconfig
+      - ~/.dotfiles/gitconfig.user:/root/.gitconfig.user
+      - ~/.ssh/id_rsa:/root/.ssh/id_rsa
+      - ~/.gem:/root/.gem
+
+volumes:
+  fog:
+  shipitron-home:
+  bundler-data:

data/lib/shipitron.rb

@@ -7,6 +7,8 @@ require 'shipitron/smash'
 module Shipitron
   CONFIG_FILE = 'shipitron/config.yml'.freeze
   SECRETS_FILE = '~/.config/shipitron/secrets.yml'.freeze
+  GLOBAL_CONFIG_FILE = '~/.config/shipitron/config.yml'.freeze
+  DEPLOY_BUCKET_PREFIX = "deploys/"
 
   class << self
     def config_file
@@ -18,10 +20,10 @@ module Shipitron
     end
 
     def config
-      @config ||= Smash.load(Pathname.new(config_file).expand_path.to_s).merge(secrets)
+      @config ||= Smash.load(Pathname.new(config_file).expand_path.to_s).merge(secrets).merge(global_config)
     rescue ArgumentError
       Logger.warn "Config file '#{config_file}' does not exist"
-      @config = secrets
+      @config = secrets.merge(global_config)
     end
 
     def secrets_file
@@ -38,5 +40,20 @@ module Shipitron
       Logger.warn "Secrets file '#{secrets_file}' does not exist"
       @secrets = Smash.new
     end
+
+    def global_config_file
+      @global_config_file ||= GLOBAL_CONFIG_FILE
+    end
+
+    def global_config_file=(file)
+      @global_config_file = file
+    end
+
+    def global_config
+      @global_config ||= Smash.load(Pathname.new(global_config_file).expand_path.to_s)
+    rescue ArgumentError
+      Logger.warn "Global config file '#{global_config_file}' does not exist"
+      @global_config = Smash.new
+    end
   end
 end

data/lib/shipitron/cli.rb

@@ -12,22 +12,26 @@ module Shipitron
     desc 'deploy <app>', 'Deploys the app'
     option :config_file, default: 'shipitron/config.yml'
     option :secrets_file, default: '~/.config/shipitron/secrets.yml'
+    option :global_config_file, default: '~/.config/shipitron/config.yml'
     option :debug, type: :boolean, default: false
     option :simulate, type: :boolean, default: false
+    option :simulate_store_deploy, type: :boolean, default: false, desc: "Simulate and store deploy config in S3"
     def deploy(app)
       setup(
         config_file: options[:config_file],
-        secrets_file: options[:secrets_file]
+        secrets_file: options[:secrets_file],
+        global_config_file: options[:global_config_file]
       )
 
       require 'shipitron/client/deploy_application'
       result = Client::DeployApplication.call(
         application: app,
-        simulate: options[:simulate]
+        simulate: options[:simulate],
+        simulate_store_deploy: options[:simulate_store_deploy]
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -50,7 +54,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -59,45 +63,40 @@ module Shipitron
 
 
     desc 'server_deploy', 'Server-side component of deploy'
-    option :name, required: true
-    option :repository, required: true
-    option :repository_branch, default: 'master'
-    option :bucket, required: true
-    option :build_cache_location, default: 'tmp/build-cache.tar.gz'
-    option :image_name, required: true
-    option :named_tag, default: 'latest'
-    option :region, required: true
-    option :clusters, type: :array, required: true
-    option :ecs_task_defs, type: :array, required: true
-    option :ecs_task_def_templates, type: :array, default: []
-    option :ecs_services, type: :array, default: []
-    option :ecs_service_templates, type: :array, default: []
-    option :build_script, default: nil
-    option :post_builds, type: :array
-    option :secrets_file, default: '~/.config/shipitron/secrets.yml'
-    option :debug, type: :boolean, default: false
+    option :deploy_id, required: true
     def server_deploy
-      setup(
-        secrets_file: options[:secrets_file]
-      )
+      setup
+
+      if !ENV.key?("SHIPITRON_DEPLOY_BUCKET") || !ENV.key?("SHIPITRON_DEPLOY_BUCKET_REGION")
+        raise "Missing shipitron deploy bucket env vars!"
+      end
+
+      require 'shipitron/server/fetch_deploy'
+      deploy_options = Server::FetchDeploy.call!(
+        deploy_bucket: ENV["SHIPITRON_DEPLOY_BUCKET"],
+        deploy_bucket_region: ENV["SHIPITRON_DEPLOY_BUCKET_REGION"],
+        deploy_id: options[:deploy_id]
+      ).deploy_options
 
       require 'shipitron/server/transform_cli_args'
       cli_args = Server::TransformCliArgs.call!(
-        application: options[:name],
-        repository_url: options[:repository],
-        repository_branch: options[:repository_branch],
-        s3_cache_bucket: options[:bucket],
-        build_cache_location: options[:build_cache_location],
-        image_name: options[:image_name],
-        named_tag: options[:named_tag],
-        region: options[:region],
-        clusters: options[:clusters],
-        ecs_task_defs: options[:ecs_task_defs],
-        ecs_task_def_templates: options[:ecs_task_def_templates],
-        ecs_services: options[:ecs_services],
-        ecs_service_templates: options[:ecs_service_templates],
-        build_script: options[:build_script],
-        post_builds: options[:post_builds]
+        application: deploy_options[:name],
+        repository_url: deploy_options[:repository],
+        repository_branch: deploy_options[:repository_branch],
+        registry: deploy_options[:registry],
+        s3_cache_bucket: deploy_options[:bucket],
+        build_cache_location: deploy_options[:build_cache_location],
+        image_name: deploy_options[:image_name],
+        named_tag: deploy_options[:named_tag],
+        skip_push: deploy_options[:skip_push],
+        region: deploy_options[:region],
+        clusters: deploy_options[:clusters],
+        ecs_task_defs: deploy_options[:ecs_task_defs],
+        ecs_task_def_templates: deploy_options[:ecs_task_def_templates],
+        ecs_services: deploy_options[:ecs_services],
+        ecs_service_templates: deploy_options[:ecs_service_templates],
+        build_script: deploy_options[:build_script],
+        post_builds: deploy_options[:post_builds]
       ).cli_args
 
       require 'shipitron/server/deploy_application'
@@ -106,7 +105,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Deploy failed.'
@@ -137,7 +136,7 @@ module Shipitron
       )
 
       if result.failure?
-        result.errors.each do |error|
+        result.error_messages.each do |error|
           Logger.fatal error
         end
         Logger.fatal 'Bootstrap failed.'
@@ -145,7 +144,7 @@ module Shipitron
     end
 
     private
-    def setup(config_file:nil, secrets_file:nil)
+    def setup(config_file:nil, secrets_file:nil, global_config_file:nil)
       $stdout.sync = true
       if options[:debug] == false
         Logger.level = :info
@@ -153,6 +152,7 @@ module Shipitron
 
       Shipitron.config_file = config_file unless config_file.nil?
       Shipitron.secrets_file = secrets_file unless secrets_file.nil?
+      Shipitron.global_config_file = global_config_file unless global_config_file.nil?
     end
   end
 end

data/lib/shipitron/client.rb

@@ -0,0 +1,9 @@
+require 'shipitron'
+
+module Shipitron
+  module Client
+    def self.started_by
+      ENV.fetch("SHIPITRON_STARTED_BY", "shipitron")
+    end
+  end
+end

data/lib/shipitron/client/bootstrap_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/register_ecs_task_definitions'
 require 'shipitron/client/create_ecs_services'
 

data/lib/shipitron/client/create_ecs_services.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 require 'securerandom'
@@ -29,13 +30,12 @@ module Shipitron
 
           service_def = Smash.load(
             path.to_s,
-            parser: MustacheYamlParser.new(
-              context: {
-                cluster: cluster_name,
-                revision: nil, # ECS will default to latest ACTIVE
-                count: service_count
-              }
-            )
+            parser: MustacheYamlParser,
+            context: {
+              cluster: cluster_name,
+              revision: nil, # ECS will default to latest ACTIVE
+              count: service_count
+            }
           ).merge(
             client_token: SecureRandom.uuid
           )

data/lib/shipitron/client/deploy_application.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/load_templates'
 require 'shipitron/client/fetch_clusters'
@@ -13,6 +14,7 @@ module Shipitron
 
       required :application
       optional :simulate
+      optional :simulate_store_deploy
 
       organize [
         LoadApplicationConfig,

data/lib/shipitron/client/ensure_deploy_not_running.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 
 # Note: This is a best effort client side check to make sure there
@@ -21,7 +22,7 @@ module Shipitron
             begin
               response = ecs_client(region: cluster.region).list_tasks(
                 cluster: cluster.name,
-                started_by: 'shipitron',
+                started_by: Shipitron::Client.started_by,
                 max_results: 1,
                 desired_status: status
               )

data/lib/shipitron/client/fetch_clusters.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'resolv'
 
 module Shipitron

data/lib/shipitron/client/force_deploy.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/client/load_application_config'
 require 'shipitron/client/fetch_clusters'

data/lib/shipitron/client/generate_deploy.rb

@@ -0,0 +1,41 @@
+require 'shipitron'
+require 'shipitron/client'
+require 'shipitron/fetch_bucket'
+require 'aws-sdk-s3'
+require 'pastel'
+
+module Shipitron
+  module Client
+    class GenerateDeploy
+      include Metaractor
+
+      required :server_deploy_opts
+      required :deploy_id
+
+      def call
+        s3_key = "#{Shipitron::DEPLOY_BUCKET_PREFIX}#{context.deploy_id}"
+        pastel = Pastel.new
+        Logger.info "Uploading deploy config to #{pastel.blue("s3://#{deploy_bucket}/#{s3_key}")}"
+
+        client = Aws::S3::Client.new(region: deploy_bucket_region)
+
+        client.put_object(
+          bucket: deploy_bucket,
+          key: s3_key,
+          body: context.server_deploy_opts.to_json,
+          acl: "private"
+        )
+      end
+
+      private
+
+      def deploy_bucket
+        Shipitron.config.deploy_bucket
+      end
+
+      def deploy_bucket_region
+        Shipitron.config.deploy_bucket_region
+      end
+    end
+  end
+end

data/lib/shipitron/client/load_application_config.rb

@@ -1,5 +1,7 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/post_build'
+require 'aws-sdk-core'
 
 module Shipitron
   module Client
@@ -11,6 +13,7 @@ module Shipitron
       def call
         context.repository_url = config.repository
         context.repository_branch = config.repository_branch
+        context.registry = config.registry
         context.s3_cache_bucket = config.cache_bucket
         context.build_cache_location = config.build_cache_location
         context.image_name = config.image_name
@@ -21,6 +24,7 @@ module Shipitron
                                 config.named_tag
                               end
                             end
+        context.skip_push = config.skip_push
         context.build_script = config.build_script
         context.post_builds = begin
                                 if config.post_builds.nil?
@@ -35,6 +39,19 @@ module Shipitron
         context.ecs_services = config.ecs_services
         context.ecs_task_def_dir = config.ecs_task_def_dir
         context.ecs_service_dir = config.ecs_service_dir
+
+        if Shipitron.config.aws_access_key_id? && Shipitron.config.aws_secret_access_key?
+          Aws.config.update(
+            credentials: Aws::Credentials.new(
+              Shipitron.config.aws_access_key_id,
+              Shipitron.config.aws_secret_access_key
+            )
+          )
+        end
+
+        if !Shipitron.config.deploy_bucket? || !Shipitron.config.deploy_bucket_region?
+          raise "Missing required deploy bucket configuration!"
+        end
       end
 
       private

data/lib/shipitron/client/load_templates.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 
 module Shipitron
   module Client

data/lib/shipitron/client/register_ecs_task_definitions.rb

@@ -1,4 +1,5 @@
 require 'shipitron'
+require 'shipitron/client'
 require 'shipitron/ecs_client'
 require 'shipitron/mustache_yaml_parser'
 
@@ -26,11 +27,10 @@ module Shipitron
 
           task_def = Smash.load(
             path.to_s,
-            parser: MustacheYamlParser.new(
-              context: {
-                tag: 'latest'
-              }
-            )
+            parser: MustacheYamlParser,
+            context: {
+              tag: 'latest'
+            }
           )
 
           begin