biosphere 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6e587744cb4b79991175f86016bcd023fce26456
-  data.tar.gz: e7b5a2b107cf724d4cc7f319528e55daf5396c84
+  metadata.gz: 18516adcf232287623fbc3b0a38a9fcbf3a06467
+  data.tar.gz: 25b96568a2cbbf6760d4cf8f5ac83c74d0111272
 SHA512:
-  metadata.gz: cd23b2e88b6a75729ec8e18b3e2e39929d526ecbc07ea87d0c710639c610d191e3b225c0876ad93a487e00489103e07bedb6e2100bf3b82c697ae3b6a33287e3
-  data.tar.gz: 85e154e3075e05572e3ea6b5d35d7591a8a3b302502044e3bc0479b0e3f1dc5078760043d5061edbcca62bbb41256ad0c506d4af5984cd3d9ffa8a35b35b5b7f
+  metadata.gz: a00f8ba95c94a69a16aad64144dbec2e5fecccb40a3a9078110ff3fb73215f0150a2822c1a0b395cac4234072f5fa7f239bfbda0cfec704bf5931367deb7c3ca
+  data.tar.gz: 1a34d3f63840b2cd4990d686d09cdc0e8cffc10084e35e176f0d1eac01967368b58acc702905c6ab2c68a845585ff86533869e9970e22c4cd34976e96e84431a

data/bin/biosphere

@@ -203,8 +203,23 @@ elsif ARGV[0] == "commit" && options.src
 
     s3.set_lock()
     s3.retrieve("#{options.build_dir}/#{deployment}.tfstate")
-    tf_plan = %x( terraform plan -state=#{options.build_dir}/#{deployment}.tfstate #{options.build_dir} )
-    puts "\n" + tf_plan
+    tf_plan_str = %x( terraform plan -state=#{options.build_dir}/#{deployment}.tfstate -out #{options.build_dir}/plan #{options.build_dir}  )
+    tfplanning = Biosphere::CLI::TerraformPlanning.new()
+    plan = tfplanning.generate_plan(suite.deployments[deployment], tf_plan_str)
+    if !plan
+        STDERR.puts "Error parsing tf plan output" 
+        s3.release_lock()
+        exit
+    end
+
+    # Print the raw terraform output
+    puts "\n" + tf_plan_str
+
+    # Print our pretty short plan
+    plan.print
+
+    targets = plan.get_resources.collect { |x| "-target=#{x}" }.join(" ")
+
     answer = ""
     while answer.empty? || (answer != "y" && answer != "n")
         print "\nDoes the plan look reasonable? (Answering yes will apply the changes) y/n: "
@@ -219,7 +234,7 @@ elsif ARGV[0] == "commit" && options.src
         #tf_apply = %x( terraform apply -state=#{state_file} #{options.build_dir})
         #puts "\n" + tf_apply
         begin
-            PTY.spawn("terraform apply -state=#{state_file} #{options.build_dir}") do |stdout, stdin, pid|
+            PTY.spawn("terraform apply #{targets} -state=#{state_file} #{options.build_dir}") do |stdout, stdin, pid|
             begin
                 stdout.each { |line| puts line }
                 rescue Errno::EIO

data/lib/biosphere.rb

@@ -11,5 +11,6 @@ require "biosphere/state"
 require "biosphere/deployment"
 require "biosphere/terraformproxy"
 require "biosphere/suite"
+require "biosphere/cli/terraformplanning"
 require "biosphere/s3"
 

data/lib/biosphere/cli/terraformplanning.rb

@@ -0,0 +1,250 @@
+require 'pp'
+require 'treetop'
+require 'colorize'
+
+class Biosphere
+
+    class CLI
+
+        class TerraformPlanning
+
+            class TerraformPlan
+                attr_accessor :items
+                def initialize()
+                    @items = []
+                end
+
+                def length
+                    @items.length
+                end
+
+                def print(out = STDOUT)
+                    last_target_group = nil
+                    @items.sort_by { |a| a[:target_group] }.each do |item|
+                        if last_target_group != item[:target_group]
+                            if item[:target_group] != ""
+                                out.write "\nTarget group: #{item[:target_group]}\n"
+                            else
+                                out.write "\nNot in any target groups:\n"
+                            end
+                            last_target_group = item[:target_group]
+                        end
+                        out.write "\t"
+                        out.write "#{item[:resource_name]} "
+                        if item[:action] == :create
+                            out.write "(#{item[:reason]})".colorize(:green)
+                        elsif item[:action] == :relaunch
+                            out.write "(#{item[:reason]})".colorize(:yellow)
+                        elsif item[:action] == :destroy
+                            out.write "(#{item[:reason]})".colorize(:red)
+                        elsif item[:action] == :change
+                            out.write "(#{item[:reason]})".colorize(:yellow)
+                        else
+                            out.write "(#{item[:reason]}) unknown action: #{item[:action]}".colorize(:red)
+                        end
+                        out.write "\n"
+                    end
+
+                    if has_unpicked_resources()
+
+                        out.write "\nWARNING: Not all resource changes will be applied!".colorize(:yellow)
+                        out.write "\nYou need to do \"biosphere commit\" again when you have verified that it is safe to do so!".colorize(:yellow)
+                    end
+                end
+
+                def has_unpicked_resources()
+                    @items.find_index { |x| x[:action] == :not_picked } != nil
+                end
+
+                def get_resources()
+                    @items.select { |x| x[:action] != :not_picked }.collect { |x| x[:resource_name] }
+                end
+            end
+
+            def generate_plan(deployment, tf_output_str)
+                data = parse_terraform_plan_output(tf_output_str)
+
+                plan = build_terraform_targetting_plan(deployment, data)
+
+                return plan
+            end
+
+            # returns object which contains interesting information on
+            # the terraform plan output.
+            #
+            # :relaunches contains a list of resources which will be changed
+            # by a relaunch
+            def parse_terraform_plan_output(str)
+                relaunches = []
+                changes = []
+                new_resources = []
+                lines = str.split("\n")
+                lines.each do |line|
+                    # the gsub is to strip possible ansi colors away
+                    # the match is to pick the TF notation about how the resource is about to change following the resource name itself
+                    m = line.gsub(/\e\[[0-9;]*m/, "").match(/^([-~+\/]+)\s(.+)$/)
+                    if m
+                        # If the resource action contains a minus ('-' or '-/+') then
+                        # we know that the action will be destructive.
+                        if m[1].match(/[-]/)
+                            relaunches << m[2]
+                        elsif m[1] == "~"
+                            changes << m[2]
+                        elsif m[1] == "+"
+                            new_resources << m[2]
+                        end
+                    end
+                end
+                return {
+                    :relaunches => relaunches,
+                    :changes => changes,
+                    :new_resources => new_resources,
+                }
+            end
+
+            def build_terraform_targetting_plan(deployment, changes)
+
+                # This function will output an array of objects which describe a proper and safe
+                # plan for terraform resources to be applied.
+                #
+                # We can include following sets in this array:
+                # - resources which do not belong to any group
+                # - resources which belong to a group where count(group) == 1
+                # - a single resource from each group where count(group) > 1
+                #
+                # Each item is an object with the following fields:
+                # - :resource_name
+                # - :target_group (might be null)
+                # - :reason (human readable reason)
+                # - :action (symbol :not_picked, :relaunch, :change, :create, :destroy)
+                #
+                plan = TerraformPlan.new()
+
+                group_changes_map = {}
+                resource_to_target_group_map = {}
+
+                resources_not_in_any_target_group = {}
+                deployment.resources.each do |resource|
+                    belong_to_target_group = false
+                    resource_name = resource[:type] + "." + resource[:name]
+
+                    deployment.target_groups.each do |group_name, resources|
+                        if resources.include?(resource_name)
+                            resource_to_target_group_map[resource_name] = group_name
+                            belong_to_target_group = true
+                        end
+                    end
+
+                    if !belong_to_target_group
+                        resources_not_in_any_target_group[resource_name] = {
+                            :resource_name => resource_name,
+                            :target_group => "",
+                            :reason => :no_target_group,
+                            :action => :relaunch
+                        }
+                    end
+                end
+
+                # Easy case first: new resources. We just want to lookup the group so that we can show that to the user
+                changes[:new_resources].each do |change|
+                    group = resource_to_target_group_map[change]
+                    if group
+                        plan.items << {
+                            :resource_name => change,
+                            :target_group => group,
+                            :reason => "new resource",
+                            :action => :create
+                        }
+                    else
+                        plan.items << {
+                            :resource_name => change,
+                            :target_group => "",
+                            :reason => "new resource",
+                            :action => :create
+                        }
+                        
+                    end
+                end
+
+                # Easy case first: new resources. We just want to lookup the group so that we can show that to the user
+                changes[:changes].each do |change|
+                    group = resource_to_target_group_map[change]
+                    if group
+                        plan.items << {
+                            :resource_name => change,
+                            :target_group => group,
+                            :reason => "non-destructive change",
+                            :action => :change
+                        }
+                    else
+                        plan.items << {
+                            :resource_name => change,
+                            :target_group => "",
+                            :reason => "non-destructive change",
+                            :action => :change
+                        }
+                        
+                    end
+                end
+
+                # Relaunches are more complex: we need to bucket resources based on group, so that we can later pick just one change from each group
+                changes[:relaunches].each do |change|
+                    group = resource_to_target_group_map[change]
+                    if group
+                        group_changes_map[group] = (group_changes_map[group] ||= []) << change
+                    elsif resources_not_in_any_target_group[change]
+                        # this handles a change to a resource which does not belong to any target group
+                        plan.items << resources_not_in_any_target_group[change]
+                    else
+                        # this handles the case where a resource was removed from the definition and
+                        # now terraform wants to destroy this resource
+                        plan.items << {
+                            :resource_name => change,
+                            :target_group => "",
+                            :reason => "resource definition has been removed",
+                            :action => :destroy
+                        }
+                        
+                    end
+                end
+
+                # Handle safe groups: just one changing resource in the group
+                safe_groups = group_changes_map.select { |name, resources| resources.length <= 1 }
+                safe_groups.each do |group_name, resources|
+                    resources.each do |resource_name|                       
+                        plan.items << {
+                            :resource_name => resource_name,
+                            :target_group => group_name,
+                            :reason => "only member in its group",
+                            :action => :relaunch
+                        }
+                    end
+                end
+
+                # Handle problematic groups: select one from each group where count(group) > 1
+                problematic_groups = group_changes_map.select { |name, resources| resources.length > 1 }
+                problematic_groups.each do |group_name, resources|
+                    original_length = resources.length
+                    plan.items << {
+                        :resource_name => resources.shift,
+                        :target_group => group_name,
+                        :reason => "group has total #{original_length} resources. Picked this as the first",
+                        :action => :relaunch
+                    }
+
+                    resources.each do |resource_name|
+                        plan.items << {
+                            :resource_name => resource_name,
+                            :target_group => group_name,
+                            :reason => "not selected from this group",
+                            :action => :not_picked
+                        }
+                    end
+                    
+                end
+
+                return plan
+            end
+        end
+    end
+end

data/lib/biosphere/deployment.rb

@@ -5,7 +5,7 @@ class Biosphere
 
     class Deployment
 
-        attr_reader :export, :name, :_settings, :feature_manifests
+        attr_reader :export, :name, :_settings, :feature_manifests, :target_groups, :resources
         attr_accessor :state, :node
         def initialize(*args)
 
@@ -56,6 +56,7 @@ class Biosphere
             @actions = {}
             @deployments = []
             @outputs = []
+            @target_groups = {}
 
             if @feature_manifests
                 node[:feature_manifests] = @feature_manifests
@@ -65,6 +66,11 @@ class Biosphere
 
         end
 
+        def add_resource_to_target_group(resource_type, resource_name, target_group)
+            name = resource_type + "." + resource_name
+            (@target_groups[target_group] ||= []) << name
+        end
+
         def setup(settings)
         end
 
@@ -103,7 +109,7 @@ class Biosphere
             @delayed << delayed_call
         end
 
-        def resource(type, name, &block)
+        def resource(type, name, target_group = nil, &block)
             if self.name
                 name = self.name + "_" + name
             end
@@ -119,10 +125,14 @@ class Biosphere
                 :location => caller[0] + "a"
             }
 
+            if target_group
+                add_resource_to_target_group(type, name, target_group)
+            end
+
             if block_given?
                 resource[:block] = block
             else
-                STDERR.puts("WARNING: No block set for resource call '#{type}', '#{name}' at #{caller[0]}")
+                #STDERR.puts("WARNING: No block set for resource call '#{type}', '#{name}' at #{caller[0]}")
             end
 
             @resources << resource
@@ -194,7 +204,9 @@ class Biosphere
             # And finish with our own resources
             @resources.each do |resource|
                 proxy = ResourceProxy.new(self)
-                proxy.instance_eval(&resource[:block])
+                if resource[:block]
+                    proxy.instance_eval(&resource[:block])
+                end
 
                 @export["resource"][resource[:type].to_s][resource[:name].to_s] = proxy.output
             end

data/lib/biosphere/version.rb

@@ -1,3 +1,3 @@
 class Biosphere
-  Version = "0.2.0"
+  Version = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: biosphere
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Juho Mäkinen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-09 00:00:00.000000000 Z
+date: 2017-05-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec
@@ -136,6 +136,20 @@ dependencies:
     - - '='
       - !ruby/object:Gem::Version
         version: 0.0.3
+- !ruby/object:Gem::Dependency
+  name: treetop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 1.6.8
 description: "Terraform's HCL lacks quite many programming features like iterators,
   true variables, advanced string manipulation, functions etc.\n\n  This Ruby tool
   provides an easy-to-use DSL to define Terraform compatible .json files which can
@@ -149,6 +163,7 @@ files:
 - bin/biosphere
 - examples/example.rb
 - lib/biosphere.rb
+- lib/biosphere/cli/terraformplanning.rb
 - lib/biosphere/deployment.rb
 - lib/biosphere/ipaddressallocator.rb
 - lib/biosphere/kube.rb