prima-twig 1.2.1 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6c85bc796545f4a0f38fa6be4380dec88e6c2fb771e8fe39d5044c57e9c0a6ee
-  data.tar.gz: 4eef557dba95dc99c88f3f07951bb6219aa2fe63eb928c12e9c6a743954400e6
+  metadata.gz: 94183c3b380822569fa755c6cd914da1eb15d36378d93a8faf827cf8b2a8a119
+  data.tar.gz: 865c21d79254b4b4ad2e8034d665f74301585fd50f4fd20b019e0c6bde977183
 SHA512:
-  metadata.gz: 4d2472dfc0c866aab6563b9cca251fd606256dc05c55bd450abbd65e5db86371327626bbb8001b06a3e15f50355f6d9a26fb1760513bc3a04698c043b9cbf572
-  data.tar.gz: 4e86ab32ba1df95732055f29da69469fdd2c71d197233c5db726e18780910141206e9eb6ab5d94d2db621b3be8199d0bf22bc69bc8fdc24c26204eef6a0f1c19
+  metadata.gz: 7604f33c35162fde446fb5e91e92451745d7eab126d09f417c36c835254949f2690c77c3f4ebbd8248c885c6c19b7786498da0cfbfe44adcb798b5641e53613f
+  data.tar.gz: 24f093ea1a89deef5debd86afdf69b5dbb43f5f1a6aef9bc68f8a8a0b9e2e938baee20ba897116d8a2d66e5e8d7b8d60a9e1f6af267973003e7357c34d138884

data/bin/twig-feature

@@ -326,7 +326,7 @@ class Release
       tags.push tag_keep_data
     end
 
-    update_cluster_stack(cluster_stack_name, tags)
+    update_cluster_stack(cluster_stack_name, tags, get_stack_parameters(cluster_stack_name))
 
     output "Finito!".green
   end
@@ -581,9 +581,9 @@ class Release
     resp.load_balancers[0].dns_name
   end
 
-  def update_cluster_stack(stack_name, tags = [])
+  def update_cluster_stack(stack_name, tags = [], parameters = [])
     stack_body = get_stack_template(stack_name)
-    update_stack(stack_name, stack_body, [], tags)
+    update_stack(stack_name, stack_body, parameters, tags)
   end
 
   def choose_branch_to_deploy(project_name, select_master = false)

data/bin/twig-packer-builder

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require_relative '../lib/prima_twig'
+require 'launchy'
+require 'yaml'
+
+class TwigPackerBuilder
+  include Command
+
+  def initialize
+    output 'Checking for new gem releases...'
+    unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
+      exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}"
+    end
+    @templates_path = "#{Dir.pwd}/amis"
+  end
+
+  def execute!(args)
+    @template_name = args[0]
+    @env = args[1]
+    @country = args[2]
+
+    File.open("#{Dir.pwd}/ami_state.yml", 'r') do |f|
+      @config = YAML.load(f.read)
+    end
+
+    @country_conf = @config['countries'].detect { |country| country['name'] == @country }
+    stop_if(@country_conf.nil?, "Cannot find country #{@country} inside ami_state.yml, exiting...".red)
+
+    @ami_conf = @country_conf['amis'].detect { |ami| ami['template'] == @template_name and ami['env'] == @env }
+    stop_if(@ami_conf.nil?, "Cannot find an ami in #{@country} generated from #{@template_name} and env #{@env} inside ami_state.yml, exiting...".red)
+
+    output "Starting update process with #{@template_name} in env #{@env} in country #{@country}".light_green
+    output 'running packer (this could take some time)'.light_green
+    new_ami_id = execute_packer
+    stop_if(new_ami_id.to_s.empty?, 'Failed to generate AMI!'.red)
+
+    output "new ami id: #{new_ami_id}".light_green
+    output "ami id that can be replaced: #{@ami_conf['id']}".light_green
+    output "you can run \"aws-vault exec YOUR_CONTRY_PROFILE -- twig-update-ami #{@ami_conf['id']} #{new_ami_id}\" to update all stacks".light_green
+    output 'Done'.light_green
+  end
+
+  private
+
+  def execute_packer
+    execute_command "AWS_MAX_ATTEMPTS=90 AWS_POLL_DELAY_SECONDS=60 packer build -var env=#{@env} -var teams_mapping=#{@ami_conf['teams_mapping']} -var country=#{@country} -var aws_account=#{@country_conf['aws_account']} -machine-readable #{@templates_path}/#{@template_name} | tee build.log"
+
+    `grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2`.sub(/\n/, '')
+  end
+
+  def help_content
+    <<-HELP
+
+    twig-packer-builder
+    ===========
+
+    Creates AMIs using umami's templates
+
+    Synopsis
+    --------
+
+      twig-packer-builder
+
+    Description
+    -----------
+
+      from umami main folder run
+      `aws-vault exec AWS_PROFILE_WITH_ACCESS_TO_COMMON_ACCOUNT -- twig-update-ami ${AMI_TEMPLATE} ${ENV} ${COUNTRY}`
+
+    Subcommand for Twig: <http://rondevera.github.io/twig/>
+    HELP
+  end
+
+  args = ARGV.dup
+
+  if args.include?('--help')
+    puts help_content
+    exit
+  end
+
+  TwigPackerBuilder.new.execute!(args)
+end

data/bin/twig-update-ami

@@ -1,166 +1,105 @@
 #!/usr/bin/env ruby
 
 require 'rubygems'
-require_relative '../lib/prima_twig.rb'
-require_relative '../lib/prima_aws_client.rb'
+require_relative '../lib/prima_twig'
+require_relative '../lib/prima_aws_client'
 require 'launchy'
-require 'json'
-require 'aws-sdk-s3'
+require 'yaml'
 
 class TwigUpdateAmi
   include Command
   include PrimaAwsClient
+
   def initialize
-    output 'Controllo se ci sono aggiornamenti da fare...'
-    exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}" unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
-    @s3 = Aws::S3::Client.new
-    @s3_bucket = 'prima-deploy'
-    @templates_base_url = "https://s3-eu-west-1.amazonaws.com"
+    output 'Checking for new gem releases...'
+    unless `gem outdated`.lines.grep(/^prima-twig \(.*\)/).empty?
+      exec "gem update prima-twig && twig update-ami #{ARGV.join ' '}"
+    end
+    @state_path = "#{Dir.pwd}/ami_state.yml"
   end
 
   def execute!(args)
-      update_amis(args[0], args[1], args[2], args[3], args[4])
-  end
-
-  private
+    old_ami = args[0]
+    new_ami = args[1]
 
-  def update_amis(ami_template, ami_id, ami_name, ami_description, env)
-    output "updating instance definition #{ami_template}".light_green
-    Dir.chdir 'ami'
-    update_instance_name(ami_id, ami_name, ami_description, ami_template)
-    output 'running packer update (this could take some time)'.light_green
-    new_ami_id = update_packer(ami_template, env)
-    # new_ami_id = 'ami-026890988d91ee8c6'
-    Dir.chdir '..'
-    stop_if(new_ami_id.to_s.empty?, 'Failed to generate AMI!'.red)
-    output "new ami id: #{new_ami_id}"
-
-    output 'searching for ami to update...'
-    ami_mappings = JSON.parse(@s3.get_object(bucket: @s3_bucket, key: "ami/ami-mappings.json")["body"].read())
-    old_amis = update_ami_mappings(ami_mappings, ami_template, env, new_ami_id)
-    stop_if(old_amis.empty?, "No ami to update! No #{ami_template} in env #{env}, exiting".yellow)
-
-    output "retrieving stacks that uses old ami ids: #{old_amis}"
-    exports = list_exports()
-    stacks = get_stacks_from_exports(exports, old_amis)
-    stop_if(stacks.empty?, "No stack to update found! This means that ami-mapping file is not in sync, please check manually")
+    output "retrieving stacks that uses old ami #{old_ami}".light_green
+    stacks = get_stacks_from_exports(list_exports, old_ami)
+    stop_if(stacks.empty?, 'No stack to update! Please check manually'.red)
 
     stacks.each do |stack|
-      output "processing stack #{stack}"
+      output "processing stack #{stack}".light_green
       if stack.include?('qa')
-        output "skipping stack #{stack} because is a qa"
+        output "skipping stack #{stack} because is a qa".yellow
         next
       else
-        stack_tags = tags_to_hashes(get_stack_tags(stack))
-        stack_tags['TemplateVersion'] = stack_tags['TemplateVersion'].to_i + 1
-
-        if stack.include?('batch')
-          stack_parameters = update_stack_parameters(get_stack_parameters(stack),
-            [
-              { parameter_key: 'AMIID', parameter_value: new_ami_id },
-              { parameter_key: 'TemplateVersion', parameter_value: stack_tags['TemplateVersion'].to_s }
-            ]
-          )
-          if stack.include?('offsite-backups')
-            stack_template = File.read("./cloudformation/stacks/batch/compute-environment-offsite-backups.yml")
-          else
-            stack_template = File.read("./cloudformation/stacks/batch/compute-environment.yml")
-          end
-        else
-          stack_parameters = update_stack_parameters(get_stack_parameters(stack),
-            [
-              { parameter_key: 'AMIID', parameter_value: new_ami_id },
-              { parameter_key: 'DesiredCapacity', parameter_value: get_desired_capacity(stack).to_s },
-              { parameter_key: 'TemplateVersion', parameter_value: stack_tags['TemplateVersion'].to_s }
-            ]
-          )
-          stack_template = File.read("./cloudformation/stacks/asg/#{stack.to_s.split("/")[1]}.yml")
+        stack_parameters = get_stack_parameters(stack)
+        stack_parameters = update_stack_parameters(stack_parameters, 'AMIID', new_ami)
+        unless stack.include?('batch')
+          stack_parameters = update_stack_parameters(stack_parameters, 'DesiredCapacity', get_desired_capacity(stack).to_s)
         end
-        update_stack(stack, stack_template, stack_parameters, hashes_to_tags(stack_tags))
+
+        update_stack_reuse_template(stack, stack_parameters)
       end
     end
 
     stacks.each do |stack|
-      if stack.include?('qa')
-        next
+      next if stack.include?('qa')
+
+      wait_for_stack_ready(stack, %w[CREATE_FAILED ROLLBACK_IN_PROGRESS ROLLBACK_FAILED DELETE_IN_PROGRESS DELETE_FAILED DELETE_COMPLETE UPDATE_ROLLBACK_FAILED UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS UPDATE_ROLLBACK_COMPLETE ROLLBACK_COMPLETE])
+    end
+
+    output 'Saving new ami in ami_state.yml'.light_green
+    File.open(@state_path, 'r') do |f|
+      @config = YAML.load(f.read)
+    end
+
+    @config['countries'].each do |country|
+      country['amis'].each do |ami|
+        ami['id'] = new_ami if ami['id'] == old_ami
       end
-      wait_for_stack_ready(stack, ['CREATE_FAILED', 'ROLLBACK_IN_PROGRESS', 'ROLLBACK_FAILED', 'DELETE_IN_PROGRESS', 'DELETE_FAILED', 'DELETE_COMPLETE', 'UPDATE_ROLLBACK_FAILED', 'UPDATE_ROLLBACK_COMPLETE_CLEANUP_IN_PROGRESS', 'UPDATE_ROLLBACK_COMPLETE', 'ROLLBACK_COMPLETE'])
     end
 
-    output 'writing new ami mapping'
-    File.open("ami/ami-mappings.json", 'w+') do |f|
-      mapping_file = JSON.pretty_generate(ami_mappings)
-      f.write(mapping_file)
-      @s3.put_object(bucket: @s3_bucket, key: "ami/ami-mappings.json", body: mapping_file)
+    File.open(@state_path, 'w+') do |f|
+      yaml = YAML.dump(@config)
+      f.write(yaml)
     end
 
-    output 'Update finished! ( ͡° ͜ʖ ͡°)'
+    output 'Update finished! ( ͡° ͜ʖ ͡°)'.light_green
   end
 
-  def get_stacks_from_exports(exports, old_amis)
+  private
+
+  def get_stacks_from_exports(exports, old_ami)
     stacks = []
-    old_amis.each do |old_ami|
-      exports.each do |export|
-        if export.value.eql?(old_ami)
-          stacks.insert(0,export.exporting_stack_id)
-        end
-      end
+    exports.each do |export|
+      stacks.insert(0, export.exporting_stack_id) if export.value.eql?(old_ami)
     end
     stacks
   end
 
-  def update_ami_mappings(mappings, ami_template, env, new_ami_id)
-    old_values = []
-    mappings.each do |item|
-       if item['ami_template'].eql?(ami_template) and item['env'].eql?(env)
-         old_values.insert(0,item['ami_id'])
-         item['ami_id'] = new_ami_id
-       end
-     end
-    old_values.uniq
-  end
-
-  def update_stack_parameters(stack_parameters, new_parameters)
-    new_parameters.each do |new_param|
-      stack_parameters.reject{ |k| k["parameter_key"] == new_param["parameter_key"] }
-      stack_parameters.push(new_param)
+  def update_stack_parameters(stack_parameters, key, value)
+    stack_parameters.each do |param|
+      param.parameter_value = value if param.parameter_key == key
     end
     stack_parameters
   end
 
-  def update_instance_name(ami_id, ami_name, ami_description, ecs_json_path)
-    ecs_json = JSON.parse File.read(ecs_json_path)
-
-    ecs_json['builders'][0]['source_ami'] = ami_id
-    ecs_json['builders'][0]['ami_name'] = ami_name
-    ecs_json['builders'][0]['ami_description'] = ami_description
-
-    File.open ecs_json_path, 'w' do |f|
-      f.write(JSON.pretty_generate(ecs_json))
-    end
-  end
-
   def get_desired_capacity(stack_name)
     stack_outputs = get_stack_outputs(stack_name)
     stack_outputs.each do |out|
-      if out.export_name.include?('EC2Fleet') or out.export_name.include?('AutoScalingGroup')
+      if out.export_name.include?('EC2Fleet') || out.export_name.include?('AutoScalingGroup') || out.export_name.include?('NodeGroup')
         return get_autoscaling_capacity(out.output_value)
       end
     end
   end
 
-  def update_packer(json_filename, env)
-    execute_command "AWS_MAX_ATTEMPTS=90 AWS_POLL_DELAY_SECONDS=60 packer build -var datadog_apikey=`biscuit get -f ../configs/secrets/common.yml common_production_apikey_datadog` -var github_token=`biscuit get -f ../configs/secrets/common.yml common_private_repo_github_token` -var drone_key=\"`biscuit get -f ../configs/secrets/common.yml drone_license_key`\" -var env=#{env} -machine-readable ./#{json_filename} | tee build.log"
-    `grep 'artifact,0,id' build.log | cut -d, -f6 | cut -d: -f2`.sub(/\n/, '')
-  end
-
   def help_content
     <<-HELP
 
     twig-update-ami
     ===========
 
-    Updates ami version and applies it to stacks on cloudformation
+    Updates AutoScalingGroups/BatchComputeEnvs with a new AMI
 
     Synopsis
     --------
@@ -170,8 +109,8 @@ class TwigUpdateAmi
     Description
     -----------
 
-      from artemide main folder run
-      `twig-update-ami ${AMI_TEMPLATE} ${AMI_ID} ${AMI_NAME} ${AMI_DESCRIPTION} ${ENV}`
+      from tsunami main folder run
+      `aws-vault exec AWS_PROFILE_WITH_ACCESS_TO_COUNTRY_ACCOUNT -- twig-update-ami ${OLD_AMI} ${NEW_AMI}`
 
     Subcommand for Twig: <http://rondevera.github.io/twig/>
     Author: Eugenio Laghi <https://github.com/eugeniolaghi>
@@ -179,13 +118,6 @@ class TwigUpdateAmi
     HELP
   end
 
-  class ::Hash
-      def deep_merge(second)
-          merger = proc { |key, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : Array === v1 && Array === v2 ? v1 | v2 : [:undefined, nil, :nil].include?(v2) ? v1 : v2 }
-          self.merge(second.to_h, &merger)
-      end
-  end
-
   args = ARGV.dup
 
   if args.include?('--help')

data/lib/prima_aws_client.rb

@@ -175,6 +175,32 @@ module PrimaAwsClient
     end
   end
 
+  def update_stack_reuse_template(stack_name, parameters = [], tags = [], role = nil)
+    cf_args = {
+      stack_name: stack_name,
+      use_previous_template: true,
+      parameters: parameters,
+      tags: tags,
+      capabilities: ['CAPABILITY_IAM']
+    }
+
+    if role != nil then
+      cf_args.merge!(role_arn: role)
+    end
+
+    begin
+      cf_client.update_stack(cf_args)
+    rescue Aws::CloudFormation::Errors::Throttling => e
+      output 'Throttling, retrying in 15 seconds'.red
+      sleep 15
+      update_stack(stack_name, template_body, parameters = [], tags = [])
+    rescue Aws::CloudFormation::Errors::ValidationError => e
+      raise e
+    else
+      output "L'update dello stack #{stack_name} è stato avviato".green
+    end
+  end
+
   def stack_exists?(stack_name)
     begin
       cf_client.describe_stacks(stack_name: stack_name)
@@ -474,21 +500,4 @@ module PrimaAwsClient
     resp = ec2_client.describe_spot_fleet_requests(spot_fleet_request_ids: [fleet_arn])
     resp.spot_fleet_request_configs[0].spot_fleet_request_config.target_capacity
   end
-
-  def hashes_to_tags(hashes)
-    tags = []
-    hkeys = hashes.keys
-    hkeys.each do |hkey|
-      tags.insert(0, { key: hkey, value: hashes[hkey].to_s })
-    end
-    tags
-  end
-
-  def tags_to_hashes(tags)
-    hash = Hash.new
-    tags.each do |tags_obj|
-      hash[tags_obj.key] = tags_obj.value
-    end
-    hash
-  end
 end

data/lib/prima_twig.rb

@@ -8,8 +8,8 @@ require 'aws-sdk-core'
 require 'rubyflare'
 
 class Prima
-  CONFIG_KEYS=['github', 'cloudflare_email', 'cloudflare_apikey', 'aws_username', 'aws_password', 'prima_apikey']
-  attr_reader :gh, :twig, :config, :rugged, :aws
+  CONFIG_KEYS=['github', 'cloudflare_email', 'cloudflare_apikey']
+  attr_reader :gh, :twig, :config, :rugged
 
   def initialize
     @twig = Twig.new(:read_options => true, :max_days_old => 30)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: prima-twig
 version: !ruby/object:Gem::Version
-  version: 1.2.1
+  version: 1.3.0
 platform: ruby
 authors:
 - Matteo Giachino
@@ -14,7 +14,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-07-21 00:00:00.000000000 Z
+date: 2020-11-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aws-sdk-autoscaling
@@ -73,21 +73,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-ec2
+  name: aws-sdk-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1'
+        version: '3'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-ecs
+  name: aws-sdk-ec2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -101,7 +101,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-elasticloadbalancingv2
+  name: aws-sdk-ecs
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -115,7 +115,7 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-s3
+  name: aws-sdk-elasticloadbalancingv2
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
@@ -129,19 +129,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1'
 - !ruby/object:Gem::Dependency
-  name: aws-sdk-core
+  name: aws-sdk-s3
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3'
+        version: '1'
 - !ruby/object:Gem::Dependency
   name: colorize
   requirement: !ruby/object:Gem::Requirement
@@ -286,11 +286,13 @@ description: Our tools to manage git and github
 email: matteo.giachino@prima.it
 executables:
 - twig-feature
+- twig-packer-builder
 - twig-update-ami
 extensions: []
 extra_rdoc_files: []
 files:
 - bin/twig-feature
+- bin/twig-packer-builder
 - bin/twig-update-ami
 - lib/command.rb
 - lib/prima_aws_client.rb