biosphere 0.2.14 → 0.2.17
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/bin/biosphere +16 -11
- data/lib/biosphere/cli/terraformplanning.rb +239 -3
- data/lib/biosphere/kube.rb +6 -2
- data/lib/biosphere/version.rb +1 -1
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 668761124d495501e57b6581a66d3a461e20e1b6
|
4
|
+
data.tar.gz: 77b077d01ee5d1b57cfa2de40fe62bfe8bb282c9
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 4fd8a19b4e2d1a045a077605d15f270170ec54f1e4dd20075704418a9dc54e7f215dc15dd1267d9876ac70f2d3e2832951abd6ebcc8b1cf853f7c4e84de62dd0
|
7
|
+
data.tar.gz: ce5820b8bb1762817df4582f24b3aeea759c9d2d90654f951331d57910d5ddd8fe2df1794db5c77881adb419f5ec123fba551a12d4a6ecad9c1d1b860f76b49b
|
data/bin/biosphere
CHANGED
@@ -273,7 +273,7 @@ elsif ARGV[0] == "statereset" && options.src
|
|
273
273
|
state.save()
|
274
274
|
s3.save("#{options.build_dir}/state.node") unless localmode
|
275
275
|
suite.deployments.each do |name, deployment|
|
276
|
-
s3.delete_object()
|
276
|
+
s3.delete_object("#{name}.tfstate")
|
277
277
|
end
|
278
278
|
end
|
279
279
|
|
@@ -302,21 +302,30 @@ elsif ARGV[0] == "commit" && options.src
|
|
302
302
|
end
|
303
303
|
|
304
304
|
s3.set_lock()
|
305
|
-
|
305
|
+
state_file = "#{options.build_dir}/#{deployment}.tfstate"
|
306
|
+
s3.retrieve(state_file)
|
306
307
|
begin
|
307
|
-
tf_plan_str = %x( terraform plan -state=#{
|
308
|
+
tf_plan_str = %x( terraform plan -state=#{state_file} #{options.build_dir}/#{deployment} )
|
308
309
|
rescue Errno::ENOENT => e
|
309
310
|
STDERR.puts "Could not find terraform. Install with with \"brew install terraform\"".colorize(:red)
|
310
311
|
s3.release_lock()
|
311
312
|
end
|
313
|
+
|
314
|
+
tf_graph_str = %x( terraform graph #{options.build_dir}/#{deployment} )
|
315
|
+
|
312
316
|
tfplanning = Biosphere::CLI::TerraformPlanning.new()
|
313
|
-
plan = tfplanning.generate_plan(suite.deployments[deployment], tf_plan_str)
|
317
|
+
plan = tfplanning.generate_plan(suite.deployments[deployment], tf_plan_str, tf_graph_str)
|
314
318
|
if !plan
|
315
319
|
STDERR.puts "Error parsing tf plan output"
|
316
320
|
s3.release_lock()
|
317
321
|
exit
|
318
322
|
end
|
319
323
|
|
324
|
+
targets = plan.get_resources.collect { |x| "-target=#{x}" }.join(" ")
|
325
|
+
puts "Targets: #{targets}"
|
326
|
+
|
327
|
+
tf_plan_str = %x( terraform plan #{targets} -state=#{state_file} -out #{options.build_dir}/plan #{options.build_dir}/#{deployment} )
|
328
|
+
|
320
329
|
# Print the raw terraform output
|
321
330
|
puts "== TERRAFORM PLAN START ==".colorize(:green)
|
322
331
|
puts "\n" + tf_plan_str
|
@@ -326,8 +335,6 @@ elsif ARGV[0] == "commit" && options.src
|
|
326
335
|
puts "Target group listing:"
|
327
336
|
plan.print
|
328
337
|
|
329
|
-
targets = plan.get_resources.collect { |x| "-target=#{x}" }.join(" ")
|
330
|
-
puts "Targets: #{targets}"
|
331
338
|
answer = ""
|
332
339
|
if !options.force
|
333
340
|
while answer.empty? || (answer != "y" && answer != "n")
|
@@ -342,11 +349,8 @@ elsif ARGV[0] == "commit" && options.src
|
|
342
349
|
puts "\nOk, will not proceed with commit"
|
343
350
|
elsif answer == "y"
|
344
351
|
puts "\nApplying the changes (this may take several minutes)"
|
345
|
-
state_file = "#{options.build_dir}/#{deployment}.tfstate"
|
346
|
-
#tf_apply = %x( terraform apply -state=#{state_file} #{options.build_dir})
|
347
|
-
#puts "\n" + tf_apply
|
348
352
|
begin
|
349
|
-
PTY.spawn("terraform apply
|
353
|
+
PTY.spawn("terraform apply -state-out=#{state_file} #{options.build_dir}/plan") do |stdout, stdin, pid|
|
350
354
|
begin
|
351
355
|
stdout.each { |line| puts line }
|
352
356
|
rescue Errno::EIO
|
@@ -375,8 +379,9 @@ elsif ARGV[0] == "commit" && options.src
|
|
375
379
|
suite.deployments[deployment].load_outputs(state_file)
|
376
380
|
state.node[:biosphere][:last_commit_time] = Time.now.utc.strftime('%Y-%m-%dT%H:%M:%SZ')
|
377
381
|
state.save()
|
378
|
-
s3.save(
|
382
|
+
s3.save(state_file)
|
379
383
|
s3.save("#{options.build_dir}/state.node")
|
384
|
+
#File.delete("#{options.build_dir}/plan")
|
380
385
|
end
|
381
386
|
|
382
387
|
s3.release_lock()
|
@@ -4,6 +4,233 @@ require 'colorize'
|
|
4
4
|
|
5
5
|
class Biosphere
|
6
6
|
|
7
|
+
class TerraformGraph
|
8
|
+
|
9
|
+
attr_accessor :graph
|
10
|
+
|
11
|
+
class Edge
|
12
|
+
attr_accessor :src, :dst, :length
|
13
|
+
|
14
|
+
def initialize(src, dst, length = 1)
|
15
|
+
@src = src
|
16
|
+
@dst = dst
|
17
|
+
@length = length
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
|
22
|
+
class Graph < Array
|
23
|
+
attr_reader :edges, :map
|
24
|
+
|
25
|
+
def initialize
|
26
|
+
@map = {}
|
27
|
+
@edges = []
|
28
|
+
end
|
29
|
+
|
30
|
+
def connect(src_name, dst_name, length = 1)
|
31
|
+
unless @map.include?(src_name)
|
32
|
+
raise ArgumentError, "No such vertex: #{src_name}"
|
33
|
+
end
|
34
|
+
unless @map.include?(dst_name)
|
35
|
+
raise ArgumentError, "No such vertex: #{dst_name}"
|
36
|
+
end
|
37
|
+
|
38
|
+
src = @map[src_name]
|
39
|
+
dst = @map[dst_name]
|
40
|
+
|
41
|
+
@edges.push Edge.new(src, dst, length)
|
42
|
+
end
|
43
|
+
|
44
|
+
def push_name(name)
|
45
|
+
if !@map[name]
|
46
|
+
index = @map.length + 1
|
47
|
+
@map[name] = index
|
48
|
+
self.push(index)
|
49
|
+
return index
|
50
|
+
else
|
51
|
+
return @map[name]
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def connect_mutually(vertex1_name, vertex2_name, length = 1)
|
56
|
+
self.connect vertex1_name, vertex2_name, length
|
57
|
+
self.connect vertex2_name, vertex1_name, length
|
58
|
+
end
|
59
|
+
|
60
|
+
def neighbors(vertex_name)
|
61
|
+
vertex = @map[vertex_name]
|
62
|
+
neighbors = []
|
63
|
+
@edges.each do |edge|
|
64
|
+
neighbors.push edge.dst if edge.src == vertex
|
65
|
+
end
|
66
|
+
return neighbors.uniq.collect { |x| @map.key(x) }
|
67
|
+
end
|
68
|
+
|
69
|
+
def neighbors_by_index(vertex)
|
70
|
+
neighbors = []
|
71
|
+
@edges.each do |edge|
|
72
|
+
neighbors.push edge.dst if edge.src == vertex
|
73
|
+
end
|
74
|
+
return neighbors.uniq
|
75
|
+
end
|
76
|
+
|
77
|
+
def length_between(src_name, dst_name)
|
78
|
+
src = @map[src_name]
|
79
|
+
dst = @map[dst_name]
|
80
|
+
|
81
|
+
@edges.each do |edge|
|
82
|
+
return edge.length if edge.src == src and edge.dst == dst
|
83
|
+
end
|
84
|
+
nil
|
85
|
+
end
|
86
|
+
|
87
|
+
def length_between_index(src, dst)
|
88
|
+
@edges.each do |edge|
|
89
|
+
return edge.length if edge.src == src and edge.dst == dst
|
90
|
+
end
|
91
|
+
nil
|
92
|
+
end
|
93
|
+
|
94
|
+
def dijkstra(src_name, dst_name = nil)
|
95
|
+
src = @map[src_name]
|
96
|
+
dst = @map[dst_name]
|
97
|
+
|
98
|
+
distances = {}
|
99
|
+
previouses = {}
|
100
|
+
self.each do |vertex|
|
101
|
+
distances[vertex] = nil # Infinity
|
102
|
+
previouses[vertex] = nil
|
103
|
+
end
|
104
|
+
distances[src] = 0
|
105
|
+
vertices = self.clone
|
106
|
+
until vertices.empty?
|
107
|
+
nearest_vertex = vertices.inject do |a, b|
|
108
|
+
next b unless distances[a]
|
109
|
+
next a unless distances[b]
|
110
|
+
next a if distances[a] < distances[b]
|
111
|
+
b
|
112
|
+
end
|
113
|
+
break unless distances[nearest_vertex] # Infinity
|
114
|
+
if dst and nearest_vertex == dst
|
115
|
+
path = path_to_names(get_path(previouses, src, dst))
|
116
|
+
return { path: path, distance: distances[dst] }
|
117
|
+
end
|
118
|
+
neighbors = vertices.neighbors_by_index(nearest_vertex)
|
119
|
+
neighbors.each do |vertex|
|
120
|
+
alt = distances[nearest_vertex] + vertices.length_between_index(nearest_vertex, vertex)
|
121
|
+
if distances[vertex].nil? or alt < distances[vertex]
|
122
|
+
distances[vertex] = alt
|
123
|
+
previouses[vertex] = nearest_vertex
|
124
|
+
# decrease-key v in Q # ???
|
125
|
+
end
|
126
|
+
end
|
127
|
+
vertices.delete nearest_vertex
|
128
|
+
end
|
129
|
+
if dst
|
130
|
+
return nil
|
131
|
+
else
|
132
|
+
paths = {}
|
133
|
+
distances.each { |k, v| paths[k] = path_to_names(get_path(previouses, src, k)) }
|
134
|
+
return { paths: paths, distances: distances }
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
private
|
139
|
+
def path_to_names(path)
|
140
|
+
p = []
|
141
|
+
path.each do |index|
|
142
|
+
p << @map.key(index)
|
143
|
+
end
|
144
|
+
|
145
|
+
return p
|
146
|
+
end
|
147
|
+
|
148
|
+
def get_path(previouses, src, dest)
|
149
|
+
path = get_path_recursively(previouses, src, dest)
|
150
|
+
path.is_a?(Array) ? path.reverse : path
|
151
|
+
end
|
152
|
+
|
153
|
+
# Unroll through previouses array until we get to source
|
154
|
+
def get_path_recursively(previouses, src, dest)
|
155
|
+
return src if src == dest
|
156
|
+
raise ArgumentError, "No path from #{src} to #{dest}" if previouses[dest].nil?
|
157
|
+
[dest, get_path_recursively(previouses, src, previouses[dest])].flatten
|
158
|
+
end
|
159
|
+
end
|
160
|
+
|
161
|
+
def initialize()
|
162
|
+
|
163
|
+
end
|
164
|
+
|
165
|
+
def parse_line(line)
|
166
|
+
if (m = line.match(/"\[(.+?)\] (?<name>\S+?)(\((.+?)\))?" \[label/))
|
167
|
+
return {
|
168
|
+
:type => :node,
|
169
|
+
:name => m[:name]
|
170
|
+
}
|
171
|
+
elsif (m = line.match(/"\[(.+?)\] (?<from>\S+?)(\s\((.+?)\)){0,1}" -> "\[(.+?)\] (?<to>\S+?)(\s\((.+?)\)){0,1}"/))
|
172
|
+
return {
|
173
|
+
:type => :edge,
|
174
|
+
:from => m[:from],
|
175
|
+
:to => m[:to]
|
176
|
+
}
|
177
|
+
else
|
178
|
+
return nil
|
179
|
+
end
|
180
|
+
end
|
181
|
+
|
182
|
+
def load(data)
|
183
|
+
|
184
|
+
@graph = Graph.new
|
185
|
+
|
186
|
+
lines = data.split("\n")
|
187
|
+
lines.each do |line|
|
188
|
+
|
189
|
+
m = parse_line(line)
|
190
|
+
if m
|
191
|
+
if m[:type] == :node
|
192
|
+
@graph.push_name m[:name]
|
193
|
+
elsif m[:type] == :edge
|
194
|
+
@graph.push_name m[:from]
|
195
|
+
@graph.push_name m[:to]
|
196
|
+
@graph.connect(m[:from], m[:to], 1)
|
197
|
+
end
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
def get_blacklist_by_dependency(item)
|
203
|
+
path = @graph.dijkstra("root", item)
|
204
|
+
return path[:path]
|
205
|
+
end
|
206
|
+
|
207
|
+
def filter_tf_plan(plan)
|
208
|
+
|
209
|
+
blacklist = []
|
210
|
+
plan.items.each do |item|
|
211
|
+
if item[:action] == :not_picked
|
212
|
+
begin
|
213
|
+
blacklist << get_blacklist_by_dependency(item[:resource_name])
|
214
|
+
rescue ArgumentError => e
|
215
|
+
puts "Error: #{e}. item: #{item}"
|
216
|
+
end
|
217
|
+
end
|
218
|
+
end
|
219
|
+
|
220
|
+
blacklist.each do |blacklist_items|
|
221
|
+
blacklist_items.each do |blacklist_path_item|
|
222
|
+
plan.items.each do |item|
|
223
|
+
if item[:resource_name] == blacklist_path_item && item[:action] != :not_picked
|
224
|
+
item[:action] = :not_picked
|
225
|
+
item[:reason] = "not selected as dependent on #{blacklist_items[blacklist_items.index(item[:resource_name])+1..-1].join(" -> ")}"
|
226
|
+
end
|
227
|
+
end
|
228
|
+
end
|
229
|
+
end
|
230
|
+
return plan
|
231
|
+
end
|
232
|
+
end
|
233
|
+
|
7
234
|
class CLI
|
8
235
|
|
9
236
|
class TerraformPlanning
|
@@ -12,6 +239,7 @@ class Biosphere
|
|
12
239
|
attr_accessor :items
|
13
240
|
def initialize()
|
14
241
|
@items = []
|
242
|
+
@graph = nil
|
15
243
|
end
|
16
244
|
|
17
245
|
def length
|
@@ -63,11 +291,20 @@ class Biosphere
|
|
63
291
|
end
|
64
292
|
end
|
65
293
|
|
66
|
-
def generate_plan(deployment, tf_output_str)
|
294
|
+
def generate_plan(deployment, tf_output_str, tf_graph_str = nil)
|
295
|
+
|
296
|
+
if tf_graph_str
|
297
|
+
@graph = Biosphere::TerraformGraph.new
|
298
|
+
@graph.load(tf_graph_str)
|
299
|
+
end
|
300
|
+
|
67
301
|
data = parse_terraform_plan_output(tf_output_str)
|
68
302
|
|
69
303
|
plan = build_terraform_targetting_plan(deployment, data)
|
70
304
|
|
305
|
+
if @graph
|
306
|
+
plan = @graph.filter_tf_plan(plan)
|
307
|
+
end
|
71
308
|
return plan
|
72
309
|
end
|
73
310
|
|
@@ -192,7 +429,6 @@ class Biosphere
|
|
192
429
|
# Relaunches are more complex: we need to bucket resources based on group, so that we can later pick just one change from each group
|
193
430
|
changes[:relaunches].each do |change|
|
194
431
|
group = resource_to_target_group_map[change]
|
195
|
-
puts "group #{group} for change #{change}"
|
196
432
|
if group
|
197
433
|
group_changes_map[group] = (group_changes_map[group] ||= []) << change
|
198
434
|
elsif resources_not_in_any_target_group[change]
|
@@ -214,7 +450,7 @@ class Biosphere
|
|
214
450
|
# Handle safe groups: just one changing resource in the group
|
215
451
|
safe_groups = group_changes_map.select { |name, resources| resources.length <= 1 }
|
216
452
|
safe_groups.each do |group_name, resources|
|
217
|
-
resources.each do |resource_name|
|
453
|
+
resources.each do |resource_name|
|
218
454
|
plan.items << {
|
219
455
|
:resource_name => resource_name,
|
220
456
|
:target_group => group_name,
|
data/lib/biosphere/kube.rb
CHANGED
@@ -84,14 +84,18 @@ class Biosphere
|
|
84
84
|
puts "Error doing api call: #{e}".colorize(:red)
|
85
85
|
puts "This might be because you did not specify namespace in your resource: #{resource[:metadata]}".colorize(:yellow)
|
86
86
|
else
|
87
|
-
puts "Error calling API: #{e}"
|
87
|
+
puts "Error calling API (on RestClient::MethodNotAllowed): #{e}"
|
88
88
|
end
|
89
89
|
puts "rest_client: #{ns_prefix + resource_name}, client: #{client.rest_client[ns_prefix + resource_name]}"
|
90
|
+
puts "Dumpin resource request:"
|
91
|
+
pp resource.to_h.to_json
|
90
92
|
raise e
|
91
93
|
|
92
94
|
rescue RestClient::Exception => e
|
93
|
-
puts "Error calling API: #{e}"
|
95
|
+
puts "Error calling API (on RestClient::Exception rescue): #{e}"
|
94
96
|
puts "rest_client: #{ns_prefix + resource_name}, client: #{client.rest_client[ns_prefix + resource_name]}"
|
97
|
+
puts "Dumpin resource request:"
|
98
|
+
pp resource.to_h.to_json
|
95
99
|
raise e
|
96
100
|
end
|
97
101
|
return {
|
data/lib/biosphere/version.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: biosphere
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.17
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Juho Mäkinen
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2017-06-
|
11
|
+
date: 2017-06-06 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: rspec
|