biosphere 0.2.14 → 0.2.17

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 503fe6f7ce0893d592c091476ad1713993007b4b
-  data.tar.gz: 2cff76a658d968277319a0f67adb2539c490dded
+  metadata.gz: 668761124d495501e57b6581a66d3a461e20e1b6
+  data.tar.gz: 77b077d01ee5d1b57cfa2de40fe62bfe8bb282c9
 SHA512:
-  metadata.gz: 2f47c9ff8a72bcb892b8e5c77c8aaf638c9db55117f8ed2bc3f0946bd77684a1e055a7ea9360ec3d7f3f8866b841fb25854d11713c3d82d4e15c2591099bd188
-  data.tar.gz: 6afcf43b0e4e4d5dd8ca2f0c2c7df4c8fa20b81fd0c6be9977031f26d8e16adf0641a444ad805b3cdaa7d6f1a149e99b89ad4573571d3f4091c4581d7adba4aa
+  metadata.gz: 4fd8a19b4e2d1a045a077605d15f270170ec54f1e4dd20075704418a9dc54e7f215dc15dd1267d9876ac70f2d3e2832951abd6ebcc8b1cf853f7c4e84de62dd0
+  data.tar.gz: ce5820b8bb1762817df4582f24b3aeea759c9d2d90654f951331d57910d5ddd8fe2df1794db5c77881adb419f5ec123fba551a12d4a6ecad9c1d1b860f76b49b

data/bin/biosphere

@@ -273,7 +273,7 @@ elsif ARGV[0] == "statereset" && options.src
         state.save()
         s3.save("#{options.build_dir}/state.node") unless localmode
         suite.deployments.each do |name, deployment|
-            s3.delete_object()
+            s3.delete_object("#{name}.tfstate")
         end
     end
 
@@ -302,21 +302,30 @@ elsif ARGV[0] == "commit" && options.src
     end
 
     s3.set_lock()
-    s3.retrieve("#{options.build_dir}/#{deployment}.tfstate")
+    state_file = "#{options.build_dir}/#{deployment}.tfstate"
+    s3.retrieve(state_file)
     begin
-        tf_plan_str = %x( terraform plan -state=#{options.build_dir}/#{deployment}.tfstate -out #{options.build_dir}/plan #{options.build_dir}/#{deployment}  )
+        tf_plan_str = %x( terraform plan -state=#{state_file} #{options.build_dir}/#{deployment}  )
     rescue Errno::ENOENT => e
         STDERR.puts "Could not find terraform. Install with with \"brew install terraform\"".colorize(:red)
         s3.release_lock()
     end
+
+    tf_graph_str = %x( terraform graph #{options.build_dir}/#{deployment} )
+
     tfplanning = Biosphere::CLI::TerraformPlanning.new()
-    plan = tfplanning.generate_plan(suite.deployments[deployment], tf_plan_str)
+    plan = tfplanning.generate_plan(suite.deployments[deployment], tf_plan_str, tf_graph_str)
     if !plan
         STDERR.puts "Error parsing tf plan output" 
         s3.release_lock()
         exit
     end
 
+    targets = plan.get_resources.collect { |x| "-target=#{x}" }.join(" ")
+    puts "Targets: #{targets}"
+
+    tf_plan_str = %x( terraform plan #{targets} -state=#{state_file} -out #{options.build_dir}/plan #{options.build_dir}/#{deployment}  )
+
     # Print the raw terraform output
     puts "== TERRAFORM PLAN START ==".colorize(:green)
     puts "\n" + tf_plan_str
@@ -326,8 +335,6 @@ elsif ARGV[0] == "commit" && options.src
     puts "Target group listing:"
     plan.print
 
-    targets = plan.get_resources.collect { |x| "-target=#{x}" }.join(" ")
-    puts "Targets: #{targets}"
     answer = ""
     if !options.force
         while answer.empty? || (answer != "y" && answer != "n")
@@ -342,11 +349,8 @@ elsif ARGV[0] == "commit" && options.src
         puts "\nOk, will not proceed with commit"
     elsif answer == "y"
         puts "\nApplying the changes (this may take several minutes)"
-        state_file = "#{options.build_dir}/#{deployment}.tfstate"
-        #tf_apply = %x( terraform apply -state=#{state_file} #{options.build_dir})
-        #puts "\n" + tf_apply
         begin
-            PTY.spawn("terraform apply #{targets} -state=#{state_file} #{options.build_dir}/#{deployment}") do |stdout, stdin, pid|
+            PTY.spawn("terraform apply -state-out=#{state_file} #{options.build_dir}/plan") do |stdout, stdin, pid|
             begin
                 stdout.each { |line| puts line }
                 rescue Errno::EIO
@@ -375,8 +379,9 @@ elsif ARGV[0] == "commit" && options.src
         suite.deployments[deployment].load_outputs(state_file)
         state.node[:biosphere][:last_commit_time] = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
         state.save()
-        s3.save("#{options.build_dir}/#{deployment}.tfstate")
+        s3.save(state_file)
         s3.save("#{options.build_dir}/state.node")
+        #File.delete("#{options.build_dir}/plan")
     end
 
     s3.release_lock()

data/lib/biosphere/cli/terraformplanning.rb

@@ -4,6 +4,233 @@ require 'colorize'
 
 class Biosphere
 
+    class TerraformGraph
+
+        attr_accessor :graph
+
+        class Edge
+            attr_accessor :src, :dst, :length
+
+            def initialize(src, dst, length = 1)
+                @src = src
+                @dst = dst
+                @length = length
+            end
+        end
+
+
+        class Graph < Array
+            attr_reader :edges, :map
+
+            def initialize
+                @map = {}
+                @edges = []
+            end
+
+            def connect(src_name, dst_name, length = 1)
+                unless @map.include?(src_name)
+                    raise ArgumentError, "No such vertex: #{src_name}"
+                end
+                unless @map.include?(dst_name)
+                    raise ArgumentError, "No such vertex: #{dst_name}"
+                end
+
+                src = @map[src_name]
+                dst = @map[dst_name]
+
+                @edges.push Edge.new(src, dst, length)
+            end
+
+            def push_name(name)
+                if !@map[name]
+                    index = @map.length + 1
+                    @map[name] = index
+                    self.push(index)
+                    return index
+                else
+                    return @map[name]
+                end
+            end
+
+            def connect_mutually(vertex1_name, vertex2_name, length = 1)
+                self.connect vertex1_name, vertex2_name, length
+                self.connect vertex2_name, vertex1_name, length
+            end
+
+            def neighbors(vertex_name)
+                vertex = @map[vertex_name]
+                neighbors = []
+                @edges.each do |edge|
+                    neighbors.push edge.dst if edge.src == vertex
+                end
+                return neighbors.uniq.collect { |x| @map.key(x) }
+            end
+
+            def neighbors_by_index(vertex)
+                neighbors = []
+                @edges.each do |edge|
+                    neighbors.push edge.dst if edge.src == vertex
+                end
+                return neighbors.uniq
+            end
+
+            def length_between(src_name, dst_name)
+                src = @map[src_name]
+                dst = @map[dst_name]
+
+                @edges.each do |edge|
+                    return edge.length if edge.src == src and edge.dst == dst
+                end
+                nil
+            end
+
+            def length_between_index(src, dst)
+                @edges.each do |edge|
+                    return edge.length if edge.src == src and edge.dst == dst
+                end
+                nil
+            end
+
+            def dijkstra(src_name, dst_name = nil)
+                src = @map[src_name]
+                dst = @map[dst_name]
+
+                distances = {}
+                previouses = {}
+                self.each do |vertex|
+                    distances[vertex] = nil # Infinity
+                    previouses[vertex] = nil
+                end
+                distances[src] = 0
+                vertices = self.clone
+                until vertices.empty?
+                    nearest_vertex = vertices.inject do |a, b|
+                        next b unless distances[a]
+                        next a unless distances[b]
+                        next a if distances[a] < distances[b]
+                        b
+                    end
+                    break unless distances[nearest_vertex] # Infinity
+                    if dst and nearest_vertex == dst
+                        path = path_to_names(get_path(previouses, src, dst))
+                        return { path: path, distance: distances[dst] }
+                    end
+                    neighbors = vertices.neighbors_by_index(nearest_vertex)
+                    neighbors.each do |vertex|
+                        alt = distances[nearest_vertex] + vertices.length_between_index(nearest_vertex, vertex)
+                        if distances[vertex].nil? or alt < distances[vertex]
+                            distances[vertex] = alt
+                            previouses[vertex] = nearest_vertex
+                            # decrease-key v in Q # ???
+                        end
+                    end
+                    vertices.delete nearest_vertex
+                end
+                if dst
+                    return nil
+                else
+                    paths = {}
+                    distances.each { |k, v| paths[k] = path_to_names(get_path(previouses, src, k)) }
+                    return { paths: paths, distances: distances }
+                end
+            end
+
+            private
+            def path_to_names(path)
+                p = []
+                path.each do |index|
+                    p << @map.key(index)
+                end
+
+                return p
+            end
+
+            def get_path(previouses, src, dest)
+                path = get_path_recursively(previouses, src, dest)
+                path.is_a?(Array) ? path.reverse : path
+            end
+
+            # Unroll through previouses array until we get to source
+            def get_path_recursively(previouses, src, dest)
+                return src if src == dest
+                raise ArgumentError, "No path from #{src} to #{dest}" if previouses[dest].nil?
+                [dest, get_path_recursively(previouses, src, previouses[dest])].flatten
+            end
+        end
+
+        def initialize()
+
+        end
+
+        def parse_line(line)
+            if (m = line.match(/"\[(.+?)\] (?<name>\S+?)(\((.+?)\))?" \[label/))
+                return {
+                    :type => :node,
+                    :name => m[:name]
+                }
+            elsif (m = line.match(/"\[(.+?)\] (?<from>\S+?)(\s\((.+?)\)){0,1}" -> "\[(.+?)\] (?<to>\S+?)(\s\((.+?)\)){0,1}"/))
+                return {
+                    :type => :edge,
+                    :from => m[:from],
+                    :to => m[:to]
+                }
+            else
+                return nil
+            end
+        end
+
+        def load(data)
+
+            @graph = Graph.new
+
+            lines = data.split("\n")
+            lines.each do |line|
+
+                m = parse_line(line)
+                if m
+                    if m[:type] == :node
+                        @graph.push_name m[:name]
+                    elsif m[:type] == :edge
+                        @graph.push_name m[:from]
+                        @graph.push_name m[:to]
+                        @graph.connect(m[:from], m[:to], 1)
+                    end
+                end
+            end
+        end
+
+        def get_blacklist_by_dependency(item)
+            path = @graph.dijkstra("root", item)
+            return path[:path]
+        end
+
+        def filter_tf_plan(plan)
+
+            blacklist = []
+            plan.items.each do |item|
+                if item[:action] == :not_picked
+                    begin
+                        blacklist << get_blacklist_by_dependency(item[:resource_name])
+                    rescue ArgumentError => e
+                        puts "Error: #{e}. item: #{item}"
+                    end
+                end
+            end
+
+            blacklist.each do |blacklist_items|
+                blacklist_items.each do |blacklist_path_item|
+                    plan.items.each do |item|
+                        if item[:resource_name] == blacklist_path_item && item[:action] != :not_picked
+                            item[:action] = :not_picked
+                            item[:reason] = "not selected as dependent on #{blacklist_items[blacklist_items.index(item[:resource_name])+1..-1].join(" -> ")}"
+                        end
+                    end
+                end
+            end
+            return plan
+        end
+    end
+
     class CLI
 
         class TerraformPlanning
@@ -12,6 +239,7 @@ class Biosphere
                 attr_accessor :items
                 def initialize()
                     @items = []
+                    @graph = nil
                 end
 
                 def length
@@ -63,11 +291,20 @@ class Biosphere
                 end
             end
 
-            def generate_plan(deployment, tf_output_str)
+            def generate_plan(deployment, tf_output_str, tf_graph_str = nil)
+
+                if tf_graph_str
+                    @graph = Biosphere::TerraformGraph.new
+                    @graph.load(tf_graph_str)
+                end
+
                 data = parse_terraform_plan_output(tf_output_str)
 
                 plan = build_terraform_targetting_plan(deployment, data)
 
+                if @graph
+                    plan = @graph.filter_tf_plan(plan)
+                end
                 return plan
             end
 
@@ -192,7 +429,6 @@ class Biosphere
                 # Relaunches are more complex: we need to bucket resources based on group, so that we can later pick just one change from each group
                 changes[:relaunches].each do |change|
                     group = resource_to_target_group_map[change]
-                    puts "group #{group} for change #{change}"
                     if group
                         group_changes_map[group] = (group_changes_map[group] ||= []) << change
                     elsif resources_not_in_any_target_group[change]
@@ -214,7 +450,7 @@ class Biosphere
                 # Handle safe groups: just one changing resource in the group
                 safe_groups = group_changes_map.select { |name, resources| resources.length <= 1 }
                 safe_groups.each do |group_name, resources|
-                    resources.each do |resource_name|                       
+                    resources.each do |resource_name|
                         plan.items << {
                             :resource_name => resource_name,
                             :target_group => group_name,

data/lib/biosphere/kube.rb

@@ -84,14 +84,18 @@ class Biosphere
                         puts "Error doing api call: #{e}".colorize(:red)
                         puts "This might be because you did not specify namespace in your resource: #{resource[:metadata]}".colorize(:yellow)
                     else 
-                        puts "Error calling API: #{e}"
+                        puts "Error calling API (on RestClient::MethodNotAllowed): #{e}"
                     end
                     puts "rest_client: #{ns_prefix + resource_name}, client: #{client.rest_client[ns_prefix + resource_name]}"
+                    puts "Dumpin resource request:"
+                    pp resource.to_h.to_json
                     raise e
 
                 rescue RestClient::Exception => e
-                    puts "Error calling API: #{e}"
+                    puts "Error calling API (on RestClient::Exception rescue): #{e}"
                     puts "rest_client: #{ns_prefix + resource_name}, client: #{client.rest_client[ns_prefix + resource_name]}"
+                    puts "Dumpin resource request:"
+                    pp resource.to_h.to_json
                     raise e
                 end
                 return {

data/lib/biosphere/version.rb

@@ -1,3 +1,3 @@
 class Biosphere
-  Version = "0.2.14"
+  Version = "0.2.17"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: biosphere
 version: !ruby/object:Gem::Version
-  version: 0.2.14
+  version: 0.2.17
 platform: ruby
 authors:
 - Juho Mäkinen
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-02 00:00:00.000000000 Z
+date: 2017-06-06 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec