prima-twig 1.2.6 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 665845adb115e9091a4d63673e3386d7b50e2043350ee342daee36122a6807f1
-  data.tar.gz: d92dc143955f44ae455eb20bead9668737183bfab7fa7f62c9b116e279c9538e
+  metadata.gz: 94183c3b380822569fa755c6cd914da1eb15d36378d93a8faf827cf8b2a8a119
+  data.tar.gz: 865c21d79254b4b4ad2e8034d665f74301585fd50f4fd20b019e0c6bde977183
 SHA512:
-  metadata.gz: 27bcc7a32bc312e359015843c05ad59872d54582ce342587894489407a6c48dbbce6997a7cbc5be2e7d246aead9dc0f235996658e450810e4409b829a670428a
-  data.tar.gz: 2474944753cd11b793107b6b8de274e7460ea2f54a2587ee43f3bef89ca2ef2278b40f861e5198a79c4fb8acce32a1e14c6bc684b479e0a5b188417b2c939462
+  metadata.gz: 7604f33c35162fde446fb5e91e92451745d7eab126d09f417c36c835254949f2690c77c3f4ebbd8248c885c6c19b7786498da0cfbfe44adcb798b5641e53613f
+  data.tar.gz: 24f093ea1a89deef5debd86afdf69b5dbb43f5f1a6aef9bc68f8a8a0b9e2e938baee20ba897116d8a2d66e5e8d7b8d60a9e1f6af267973003e7357c34d138884

data/bin/twig-packer-builder

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require_relative '../lib/prima_twig'
+require 'launchy'
+require 'yaml'
+
+class TwigPackerBuilder
+  include Command
+
+  def initialize
+    output 'Checking for new gem releases...'
+    unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
+      exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}"
+    end
+    @templates_path = "#{Dir.pwd}/amis"
+  end
+
+  def execute!(args)
+    @template_name = args[0]
+    @env = args[1]
+    @country = args[2]
+
+    File.open("#{Dir.pwd}/ami_state.yml", 'r') do |f|
+      @config = YAML.load(f.read)
+    end
+
+    @country_conf = @config['countries'].detect { |country| country['name'] == @country }
+    stop_if(@country_conf.nil?, "Cannot find country #{@country} inside ami_state.yml, exiting...".red)
+
+    @ami_conf = @country_conf['amis'].detect { |ami| ami['template'] == @template_name and ami['env'] == @env }
+    stop_if(@ami_conf.nil?, "Cannot find an ami in #{@country} generated from #{@template_name} and env #{@env} inside ami_state.yml, exiting...".red)
+
+    output "Starting update process with #{@template_name} in env #{@env} in country #{@country}".light_green
+    output 'running packer (this could take some time)'.light_green
+    new_ami_id = execute_packer
+    stop_if(new_ami_id.to_s.empty?, 'Failed to generate AMI!'.red)
+
+    output "new ami id: #{new_ami_id}".light_green
+    output "ami id that can be replaced: #{@ami_conf['id']}".light_green
+    output "you can run \"aws-vault exec YOUR_CONTRY_PROFILE -- twig-update-ami #{@ami_conf['id']} #{new_ami_id}\" to update all stacks".light_green
+    output 'Done'.light_green
+  end
+
+  private
+
+  def execute_packer
+    execute_command "AWS_MAX_ATTEMPTS=90 AWS_POLL_DELAY_SECONDS=60 packer build -var env=#{@env} -var teams_mapping=#{@ami_conf['teams_mapping']} -var country=#{@country} -var aws_account=#{@country_conf['aws_account']} -machine-readable #{@templates_path}/#{@template_name} | tee build.log"
+
+    `grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2`.sub(/\n/, '')
+  end
+
+  def help_content
+    <<-HELP
+
+    twig-packer-builder
+    ===========
+
+    Creates AMIs using umami's templates
+
+    Synopsis
+    --------
+
+      twig-packer-builder
+
+    Description
+    -----------
+
+      from umami main folder run
+      `aws-vault exec AWS_PROFILE_WITH_ACCESS_TO_COMMON_ACCOUNT -- twig-update-ami ${AMI_TEMPLATE} ${ENV} ${COUNTRY}`
+
+    Subcommand for Twig: <http://rondevera.github.io/twig/>
+    HELP
+  end
+
+  args = ARGV.dup
+
+  if args.include?('--help')
+    puts help_content
+    exit
+  end
+
+  TwigPackerBuilder.new.execute!(args)
+end

data/bin/twig-update-ami

@@ -1,164 +1,105 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require_relative '../lib/prima_twig.rb'
-require_relative '../lib/prima_aws_client.rb'
+require_relative '../lib/prima_twig'
+require_relative '../lib/prima_aws_client'
 require 'launchy'
-require 'json'
-require 'aws-sdk-s3'
+require 'yaml'
 
 class TwigUpdateAmi
   include Command
   include PrimaAwsClient
+
   def initialize
-    output 'Controllo se ci sono aggiornamenti da fare...'
-    exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}" unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
-    @s3 = Aws::S3::Client.new
-    @s3_bucket = 'prima-deploy'
-    @templates_base_url = "https://s3-eu-west-1.amazonaws.com"
+    output 'Checking for new gem releases...'
+    unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
+      exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}"
+    end
+    @state_path = "#{Dir.pwd}/ami_state.yml"
   end
 
   def execute!(args)
-      update_amis(args[0], args[1], args[2], args[3], args[4])
-  end
-
-  private
-
-  def update_amis(ami_template, ami_id, ami_name, ami_description, env)
-    output "updating instance definition #{ami_template}".light_green
-    Dir.chdir 'ami'
-    update_instance_name(ami_id, ami_name, ami_description, ami_template)
-    output 'running packer update (this could take some time)'.light_green
-    ami_mappings = JSON.parse(@s3.get_object(bucket: @s3_bucket, key: "ami/ami-mappings.json")["body"].read())
-
-    ami_mapping = nil
-    ami_mappings.each do |item|
-      if item['ami_template'].eql?(ami_template) and item['env'].eql?(env)
-        ami_mapping = item
-      end
-    end
-
-    stop_if(ami_mapping == nil, "No AMI found with parameters #{ami_template} and #{env}")
+    old_ami = args[0]
+    new_ami = args[1]
 
-    new_ami_id = update_packer(ami_template, env, ami_mapping['teams_mapping'], ami_mapping['country'])
-    # new_ami_id = 'ami-0d8488b68731e8756'
-    Dir.chdir '..'
-    stop_if(new_ami_id.to_s.empty?, 'Failed to generate AMI!'.red)
-    output "new ami id: #{new_ami_id}"
-
-    output 'searching for ami to update...'
-    old_amis = update_ami_mappings(ami_mappings, ami_template, env, new_ami_id)
-    stop_if(old_amis.empty?, "No ami to update! No #{ami_template} in env #{env}, exiting".yellow)
-
-    output "retrieving stacks that uses old ami ids: #{old_amis}"
-    exports = list_exports()
-    stacks = get_stacks_from_exports(exports, old_amis)
-    stop_if(stacks.empty?, "No stack to update found! This means that ami-mapping file is not in sync, please check manually")
+    output "retrieving stacks that uses old ami #{old_ami}".light_green
+    stacks = get_stacks_from_exports(list_exports, old_ami)
+    stop_if(stacks.empty?, 'No stack to update! Please check manually'.red)
 
     stacks.each do |stack|
-      output "processing stack #{stack}"
+      output "processing stack #{stack}".light_green
       if stack.include?('qa')
-        output "skipping stack #{stack} because is a qa"
+        output "skipping stack #{stack} because is a qa".yellow
         next
       else
         stack_parameters = get_stack_parameters(stack)
-        stack_parameters = update_stack_parameters(stack_parameters, "AMIID", new_ami_id)
-        if stack.include?('batch')
-          if stack.include?('offsite-backups')
-            stack_template = File.read("./cloudformation/stacks/batch/compute-environment-offsite-backups.yml")
-          else
-            stack_template = File.read("./cloudformation/stacks/batch/compute-environment.yml")
-          end
-        else
-          stack_parameters = update_stack_parameters(stack_parameters, "DesiredCapacity", get_desired_capacity(stack).to_s)
-          stack_template = File.read("./cloudformation/stacks/asg/#{stack.to_s.split("/")[1]}.yml")
+        stack_parameters = update_stack_parameters(stack_parameters, 'AMIID', new_ami)
+        unless stack.include?('batch')
+          stack_parameters = update_stack_parameters(stack_parameters, 'DesiredCapacity', get_desired_capacity(stack).to_s)
         end
-        update_stack(stack, stack_template, stack_parameters)
+
+        update_stack_reuse_template(stack, stack_parameters)
       end
     end
 
     stacks.each do |stack|
-      if stack.include?('qa')
-        next
+      next if stack.include?('qa')
+
+      wait_for_stack_ready(stack, %w[CREATE_FAILED ROLLBACK_IN_PROGRESS ROLLBACK_FAILED DELETE_IN_PROGRESS DELETE_FAILED DELETE_COMPLETE UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_ROLLBACK_COMPLETE ROLLBACK_COMPLETE])
+    end
+
+    output 'Saving new ami in ami_state.yml'.light_green
+    File.open(@state_path, 'r') do |f|
+      @config = YAML.load(f.read)
+    end
+
+    @config['countries'].each do |country|
+      country['amis'].each do |ami|
+        ami['id'] = new_ami if ami['id'] == old_ami
       end
-      wait_for_stack_ready(stack, ['CREATE_FAILED', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED', 'DELETE_IN_PROGRESS', 'DELETE_FAILED', 'DELETE_COMPLETE', 'UPDATE_ROLLBACK_FAILED', 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS', 'UPDATE_ROLLBACK_COMPLETE', 'ROLLBACK_COMPLETE'])
     end
 
-    output 'writing new ami mapping'
-    File.open("ami/ami-mappings.json", 'w+') do |f|
-      mapping_file = JSON.pretty_generate(ami_mappings)
-      f.write(mapping_file)
-      @s3.put_object(bucket: @s3_bucket, key: "ami/ami-mappings.json", body: mapping_file)
+    File.open(@state_path, 'w+') do |f|
+      yaml = YAML.dump(@config)
+      f.write(yaml)
     end
 
-    output 'Update finished! ( ͡° ͜ʖ ͡°)'
+    output 'Update finished! ( ͡° ͜ʖ ͡°)'.light_green
   end
 
-  def get_stacks_from_exports(exports, old_amis)
+  private
+
+  def get_stacks_from_exports(exports, old_ami)
     stacks = []
-    old_amis.each do |old_ami|
-      exports.each do |export|
-        if export.value.eql?(old_ami)
-          stacks.insert(0,export.exporting_stack_id)
-        end
-      end
+    exports.each do |export|
+      stacks.insert(0, export.exporting_stack_id) if export.value.eql?(old_ami)
     end
     stacks
   end
 
-  def update_ami_mappings(mappings, ami_template, env, new_ami_id)
-    old_values = []
-    mappings.each do |item|
-       if item['ami_template'].eql?(ami_template) and item['env'].eql?(env)
-         old_values.insert(0,item['ami_id'])
-         item['ami_id'] = new_ami_id
-       end
-     end
-    old_values.uniq
-  end
-
   def update_stack_parameters(stack_parameters, key, value)
     stack_parameters.each do |param|
-      if param.parameter_key == key
-        param.parameter_value = value
-      end
+      param.parameter_value = value if param.parameter_key == key
     end
     stack_parameters
   end
 
-  def update_instance_name(ami_id, ami_name, ami_description, ecs_json_path)
-    ecs_json = JSON.parse File.read(ecs_json_path)
-
-    ecs_json['builders'][0]['source_ami'] = ami_id
-    ecs_json['builders'][0]['ami_name'] = ami_name
-    ecs_json['builders'][0]['ami_description'] = ami_description
-
-    File.open ecs_json_path, 'w' do |f|
-      f.write(JSON.pretty_generate(ecs_json))
-    end
-  end
-
   def get_desired_capacity(stack_name)
     stack_outputs = get_stack_outputs(stack_name)
     stack_outputs.each do |out|
-      if out.export_name.include?('EC2Fleet') or out.export_name.include?('AutoScalingGroup') or out.export_name.include?('NodeGroup')
+      if out.export_name.include?('EC2Fleet') || out.export_name.include?('AutoScalingGroup') || out.export_name.include?('NodeGroup')
         return get_autoscaling_capacity(out.output_value)
       end
     end
   end
 
-  def update_packer(json_filename, env, teams, country)
-    execute_command "AWS_MAX_ATTEMPTS=90 AWS_POLL_DELAY_SECONDS=60 packer build -var datadog_apikey=`biscuit get -f ../configs/secrets/common.yml common_production_apikey_datadog` -var github_token=`biscuit get -f ../configs/secrets/common.yml common_private_repo_github_token` -var drone_key=\"`biscuit get -f ../configs/secrets/common.yml drone_license_key`\" -var env=#{env} -var teams_mapping=#{teams} -var country=#{country} -machine-readable ./#{json_filename} | tee build.log"
-    `grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2`.sub(/\n/, '')
-  end
-
   def help_content
     <<-HELP
 
     twig-update-ami
     ===========
 
-    Updates ami version and applies it to stacks on cloudformation
+    Updates AutoScalingGroups/BatchComputeEnvs with a new AMI
 
     Synopsis
     --------
@@ -168,8 +109,8 @@ class TwigUpdateAmi
     Description
     -----------
 
-      from artemide main folder run
-      `twig-update-ami ${AMI_TEMPLATE} ${AMI_ID} ${AMI_NAME} ${AMI_DESCRIPTION} ${ENV}`
+      from tsunami main folder run
+      `aws-vault exec AWS_PROFILE_WITH_ACCESS_TO_COUNTRY_ACCOUNT -- twig-update-ami ${OLD_AMI} ${NEW_AMI}`
 
     Subcommand for Twig: <http://rondevera.github.io/twig/>
     Author: Eugenio Laghi <https://github.com/eugeniolaghi>
@@ -177,13 +118,6 @@ class TwigUpdateAmi
     HELP
   end
 
-  class ::Hash
-      def deep_merge(second)
-          merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
-          self.merge(second.to_h, &merger)
-      end
-  end
-
   args = ARGV.dup
 
   if args.include?('--help')

data/lib/prima_aws_client.rb

@@ -175,6 +175,32 @@ module PrimaAwsClient
     end
   end
 
+  def update_stack_reuse_template(stack_name, parameters = [], tags = [], role = nil)
+    cf_args = {
+      stack_name: stack_name,
+      use_previous_template: true,
+      parameters: parameters,
+      tags: tags,
+      capabilities: ['CAPABILITY_IAM']
+    }
+
+    if role != nil then
+      cf_args.merge!(role_arn: role)
+    end
+
+    begin
+      cf_client.update_stack(cf_args)
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      update_stack(stack_name, template_body, parameters = [], tags = [])
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise e
+    else
+      output "L'update dello stack #{stack_name} è stato avviato".green
+    end
+  end
+
   def stack_exists?(stack_name)
     begin
       cf_client.describe_stacks(stack_name: stack_name)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prima-twig
 version: !ruby/object:Gem::Version
-  version: 1.2.6
+  version: 1.3.0
 platform: ruby
 authors:
 - Matteo Giachino
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-10-21 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-autoscaling
@@ -73,21 +73,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-ec2
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-ecs
+  name: aws-sdk-ec2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -101,7 +101,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-elasticloadbalancingv2
+  name: aws-sdk-ecs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -115,7 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-s3
+  name: aws-sdk-elasticloadbalancingv2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -129,19 +129,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -286,11 +286,13 @@ description: Our tools to manage git and github
 email: matteo.giachino@prima.it
 executables:
 - twig-feature
+- twig-packer-builder
 - twig-update-ami
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/twig-feature
+- bin/twig-packer-builder
 - bin/twig-update-ami
 - lib/command.rb
 - lib/prima_aws_client.rb
@@ -314,7 +316,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.1
+rubygems_version: 3.0.3
 signing_key: 
 specification_version: 4
 summary: The Prima twig toolbelt