prima-twig 0.54.46 → 0.54.230

data/bin/twig-update-ami

@@ -5,7 +5,7 @@ require_relative '../lib/prima_twig.rb'
 require_relative '../lib/prima_aws_client.rb'
 require 'launchy'
 require 'json'
-require 'aws-sdk'
+require 'aws-sdk-s3'
 
 class TwigUpdateAmi
   include Command
@@ -29,8 +29,8 @@ class TwigUpdateAmi
     Dir.chdir 'ami'
     update_instance_name(ami_id, ami_name, ami_description, ami_template)
     output 'running packer update (this could take some time)'.light_green
-    new_ami_id = update_packer(ami_template)
-    # new_ami_id = 'ami-000e38f3a919824ae'
+    new_ami_id = update_packer(ami_template, env)
+    # new_ami_id = 'ami-026890988d91ee8c6'
     Dir.chdir '..'
     stop_if(new_ami_id.to_s.empty?, 'Failed to generate AMI!'.red)
     output "new ami id: #{new_ami_id}"
@@ -50,33 +50,33 @@ class TwigUpdateAmi
       if stack.include?('qa')
         output "skipping stack #{stack} because is a qa"
         next
-      elsif stack.include?('batch')
-        stack_parameters = update_stack_parameters(get_stack_parameters(stack),
-          [
-            { parameter_key: 'AMIID', parameter_value: new_ami_id }
-          ]
-        )
-        update_stack(stack, get_stack_template(stack), stack_parameters)
-      elsif stack.include?('fleet')
+      else
         stack_tags = tags_to_hashes(get_stack_tags(stack))
         stack_tags['TemplateVersion'] = stack_tags['TemplateVersion'].to_i + 1
 
-        stack_parameters = update_stack_parameters(get_stack_parameters(stack),
-          [
-            { parameter_key: 'AMIID', parameter_value: new_ami_id },
-            { parameter_key: 'DesiredCapacity', parameter_value: get_desired_capacity(stack).to_s },
-            { parameter_key: 'TemplateVersion', parameter_value: stack_tags['TemplateVersion'].to_s }
-          ]
-        )
-        update_stack(stack, get_stack_template(stack), stack_parameters, hashes_to_tags(stack_tags))
-      else # autoscaling group normali come ania-tools
-        stack_parameters = update_stack_parameters(get_stack_parameters(stack),
-          [
-            { parameter_key: 'AMIID', parameter_value: new_ami_id },
-            { parameter_key: 'DesiredCapacity', parameter_value: get_desired_capacity(stack).to_s }
-          ]
-        )
-        update_stack(stack, get_stack_template(stack), stack_parameters)
+        if stack.include?('batch')
+          stack_parameters = update_stack_parameters(get_stack_parameters(stack),
+            [
+              { parameter_key: 'AMIID', parameter_value: new_ami_id },
+              { parameter_key: 'TemplateVersion', parameter_value: stack_tags['TemplateVersion'].to_s }
+            ]
+          )
+          if stack.include?('offsite-backups')
+            stack_template = File.read("./cloudformation/stacks/batch/compute-environment-offsite-backups.yml")
+          else
+            stack_template = File.read("./cloudformation/stacks/batch/compute-environment.yml")
+          end
+        else
+          stack_parameters = update_stack_parameters(get_stack_parameters(stack),
+            [
+              { parameter_key: 'AMIID', parameter_value: new_ami_id },
+              { parameter_key: 'DesiredCapacity', parameter_value: get_desired_capacity(stack).to_s },
+              { parameter_key: 'TemplateVersion', parameter_value: stack_tags['TemplateVersion'].to_s }
+            ]
+          )
+          stack_template = File.read("./cloudformation/stacks/asg/#{stack.to_s.split("/")[1]}.yml")
+        end
+        update_stack(stack, stack_template, stack_parameters, hashes_to_tags(stack_tags))
       end
     end
 
@@ -149,8 +149,8 @@ class TwigUpdateAmi
     end
   end
 
-  def update_packer(json_filename)
-    execute_command "packer build  -var datadog_apikey=`biscuit get -f ../configs/secrets/common.yml common_production_apikey_datadog` -var github_token=`biscuit get -f ../configs/secrets/common.yml common_private_repo_github_token` -var drone_key=\"`biscuit get -f ../configs/secrets/common.yml drone_license_key`\" -machine-readable ./#{json_filename} | tee build.log"
+  def update_packer(json_filename, env)
+    execute_command "AWS_MAX_ATTEMPTS=90 AWS_POLL_DELAY_SECONDS=60 packer build -var datadog_apikey=`biscuit get -f ../configs/secrets/common.yml common_production_apikey_datadog` -var github_token=`biscuit get -f ../configs/secrets/common.yml common_private_repo_github_token` -var drone_key=\"`biscuit get -f ../configs/secrets/common.yml drone_license_key`\" -var env=#{env} -machine-readable ./#{json_filename} | tee build.log"
     `grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2`.sub(/\n/, '')
   end
 

data/lib/prima_aws_client.rb

@@ -1,4 +1,11 @@
-require 'aws-sdk'
+require 'aws-sdk-autoscaling'
+require 'aws-sdk-batch'
+require 'aws-sdk-cloudformation'
+require 'aws-sdk-cloudfront'
+require 'aws-sdk-ec2'
+require 'aws-sdk-ecs'
+require 'aws-sdk-elasticloadbalancingv2'
+require 'aws-sdk-s3'
 require 'colorize'
 #
 module PrimaAwsClient
@@ -18,6 +25,14 @@ module PrimaAwsClient
     @ec2 ||= Aws::EC2::Client.new
   end
 
+  def alb_client
+    @alb ||= Aws::ElasticLoadBalancingV2::Client.new
+  end
+
+  def ecs_client
+    @ecs ||= Aws::ECS::Client.new
+  end
+
   def stack_list
     stacks = []
     next_token = ''
@@ -204,6 +219,24 @@ module PrimaAwsClient
     output "\nStack #{stack_name} pronto!\n".green
   end
 
+  def wait_for_stack_removal(stack_name)
+    ready = false
+    sleep_seconds = 13
+    sleep 10
+    output "Attendo che lo stack #{stack_name} finisca di essere cancellato...\n".yellow
+    while !ready
+      ready = true if stack_deleted?(stack_name)
+      seconds_elapsed = 0
+      loop do
+        break if seconds_elapsed >= sleep_seconds
+        print '.'.yellow; STDOUT.flush
+        sleep 1
+        seconds_elapsed += 1
+      end
+    end
+    output "\nStack #{stack_name} eliminato!\n".green
+  end
+
   def get_stack_tags(name)
     begin
       resp = cf_client.describe_stacks(stack_name: name)
@@ -267,6 +300,23 @@ module PrimaAwsClient
     ['CREATE_COMPLETE', 'UPDATE_COMPLETE', 'UPDATE_ROLLBACK_COMPLETE', 'ROLLBACK_COMPLETE'].include? stack_status
   end
 
+  def stack_deleted?(stack_name, failed_statuses = ['ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED', 'DELETE_FAILED', 'UPDATE_ROLLBACK_FAILED', 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS'])
+    begin
+      resp = cf_client.describe_stacks(
+        stack_name: stack_name
+      )
+      stack_status = resp.stacks[0].stack_status
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      print 'Throttling'.red; STDOUT.flush
+      return false
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      print 'Stack deleted'
+      return true
+    end
+    raise "The stack #{stack_name} errored out" if failed_statuses.include? stack_status
+    ['DELETE_COMPLETE'].include? stack_status
+  end
+
   def artifact_exists?(bucket, path)
     resp = s3_client.list_objects(
       bucket: bucket,
@@ -287,6 +337,29 @@ module PrimaAwsClient
     output "#{s3_bucket}/#{destination_path} uploadato con successo!\n".green
   end
 
+  def wait_for_artifact(bucket, path)
+    ready = artifact_exists?(bucket, path)
+    sleep_seconds = 13
+    output "Attendo che sia pronto l'artefatto #{path}...\n".yellow
+    retries = 0
+    while !ready
+      ready = true if artifact_exists?(bucket, path)
+      seconds_elapsed = 0
+      loop do
+        break if seconds_elapsed >= sleep_seconds
+        print '.'.yellow; STDOUT.flush
+        sleep 1
+        seconds_elapsed += 1
+      end
+      retries += 1
+      if retries > 150
+        output "\n Timeout raggiunto aspettando #{path}\n".red
+        exit
+      end
+    end
+    output "\nArtefatto #{path} creato!\n".green
+  end
+
   def list_import_stacks(export_name)
     stacks = []
     next_token = ''
@@ -307,6 +380,91 @@ module PrimaAwsClient
     stacks
   end
 
+  def describe_stack_resource(cluster_stack_name, logical_resource_id)
+    begin
+      resp = cf_client.describe_stack_resource({stack_name: cluster_stack_name, logical_resource_id: logical_resource_id})
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = describe_stack_resource(cluster_stack_name, logical_resource_id)
+    end
+  end
+
+  def describe_instances(instance_ids)
+    begin
+      resp = ec2_client.describe_instances({instance_ids: instance_ids})
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = describe_instances(instance_ids)
+    end
+  end
+
+  def describe_auto_scaling_groups(auto_scaling_group_names, max_records)
+    begin
+      resp = asg_client.describe_auto_scaling_groups({
+        auto_scaling_group_names: auto_scaling_group_names,
+        max_records: max_records
+      })
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = describe_auto_scaling_groups(auto_scaling_group_names, max_records)
+    end
+  end
+
+  def describe_load_balancers(load_balancer_arns)
+    begin
+      resp = alb_client.describe_load_balancers({load_balancer_arns: load_balancer_arns})
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = describe_load_balancers(load_balancer_arns)
+    end
+  end
+
+  def update_ecs_service(cluster, service, deployment_configuration)
+    begin
+      resp = ecs_client.update_service(
+        cluster: cluster,
+        service: service,
+        deployment_configuration: deployment_configuration
+      )
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = update_ecs_service(cluster, service, deployment_configuration)
+    end
+  end
+
+  def describe_ecs_tasks(cluster, tasks)
+    begin
+      resp = ecs_client.describe_tasks({
+        cluster: cluster,
+        tasks: tasks
+      })
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = describe_ecs_tasks(cluster, tasks)
+    end
+  end
+
+  def run_ecs_task(cluster, task_definition, overrides, count)
+    begin
+      resp = ecs_client.run_task({
+        cluster: cluster,
+        task_definition: task_definition,
+        overrides: overrides,
+        count: count
+      })
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      resp = run_ecs_task(cluster, task_definition, overrides, count)
+    end
+  end
+
   def get_autoscaling_capacity(asg_name)
     resp = asg_client.describe_auto_scaling_groups(auto_scaling_group_names: [asg_name])
     resp.auto_scaling_groups[0].desired_capacity

data/lib/prima_twig.rb

@@ -4,7 +4,7 @@ require 'octokit'
 require 'yaml'
 require 'rugged'
 require 'highline/import'
-require 'aws-sdk'
+require 'aws-sdk-core'
 require 'rubyflare'
 
 class Prima
@@ -20,7 +20,9 @@ class Prima
     @twig = Twig.new(:read_options => true, :max_days_old => 30)
     unless has_config?
       if File.exist?('../prima/twig.yml')
-         `cp ../prima/twig.yml .`
+        `cp ../prima/twig.yml .`
+      elsif File.exist?('/tmp/twig.yml')
+        `cp /tmp/twig.yml .`
       else
         generate_config
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prima-twig
 version: !ruby/object:Gem::Version
-  version: 0.54.46
+  version: 0.54.230
 platform: ruby
 authors:
 - Matteo Giachino
@@ -14,10 +14,122 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-11-19 00:00:00.000000000 Z
+date: 2020-06-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: aws-sdk
+  name: aws-sdk-autoscaling
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-batch
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudformation
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-cloudfront
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ec2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-ecs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-elasticloadbalancingv2
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-s3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1'
+- !ruby/object:Gem::Dependency
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -173,6 +285,7 @@ dependencies:
 description: Our tools to manage git and github
 email: matteo.giachino@prima.it
 executables:
+- twig-build
 - twig-circle
 - twig-deploy
 - twig-feature
@@ -184,6 +297,7 @@ executables:
 extensions: []
 extra_rdoc_files: []
 files:
+- bin/twig-build
 - bin/twig-circle
 - bin/twig-deploy
 - bin/twig-feature
@@ -214,7 +328,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.0.1
 signing_key: 
 specification_version: 4
 summary: The Prima twig toolbelt