pipedream 0.1.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.cody/buildspec.yml +9 -0
- data/.cody/project.rb +4 -0
- data/.gitignore +16 -11
- data/.gitmodules +9 -0
- data/.rspec +1 -1
- data/CHANGELOG.md +34 -0
- data/Gemfile +3 -1
- data/Gemfile.lock +112 -0
- data/Guardfile +19 -0
- data/LICENSE.txt +18 -17
- data/README.md +55 -21
- data/Rakefile +10 -2
- data/exe/pipe +14 -0
- data/exe/pipedream +14 -0
- data/lib/pipedream/autoloader.rb +21 -0
- data/lib/pipedream/aws_services/helpers.rb +71 -0
- data/lib/pipedream/aws_services.rb +20 -0
- data/lib/pipedream/build.rb +13 -0
- data/lib/pipedream/cli.rb +60 -0
- data/lib/pipedream/command.rb +82 -0
- data/lib/pipedream/completer/script.rb +6 -0
- data/lib/pipedream/completer/script.sh +10 -0
- data/lib/pipedream/completer.rb +159 -0
- data/lib/pipedream/core.rb +63 -0
- data/lib/pipedream/create.rb +12 -0
- data/lib/pipedream/delete.rb +27 -0
- data/lib/pipedream/deploy.rb +40 -0
- data/lib/pipedream/dsl/pipeline/approve.rb +34 -0
- data/lib/pipedream/dsl/pipeline/codebuild.rb +57 -0
- data/lib/pipedream/dsl/pipeline/github.rb +42 -0
- data/lib/pipedream/dsl/pipeline.rb +37 -0
- data/lib/pipedream/dsl/role.rb +50 -0
- data/lib/pipedream/dsl/schedule.rb +30 -0
- data/lib/pipedream/dsl/sns.rb +15 -0
- data/lib/pipedream/dsl/ssm.rb +22 -0
- data/lib/pipedream/dsl/webhook.rb +27 -0
- data/lib/pipedream/evaluate.rb +47 -0
- data/lib/pipedream/help/completion.md +22 -0
- data/lib/pipedream/help/completion_script.md +3 -0
- data/lib/pipedream/help/deploy.md +54 -0
- data/lib/pipedream/help/start.md +20 -0
- data/lib/pipedream/help.rb +9 -0
- data/lib/pipedream/init.rb +68 -0
- data/lib/pipedream/pipeline/s3_bucket.rb +88 -0
- data/lib/pipedream/pipeline.rb +61 -0
- data/lib/pipedream/role.rb +181 -0
- data/lib/pipedream/schedule.rb +99 -0
- data/lib/pipedream/sequence.rb +66 -0
- data/lib/pipedream/setting.rb +82 -0
- data/lib/pipedream/sns.rb +43 -0
- data/lib/pipedream/stack.rb +95 -0
- data/lib/pipedream/start.rb +84 -0
- data/lib/pipedream/update.rb +12 -0
- data/lib/pipedream/version.rb +1 -1
- data/lib/pipedream/webhook.rb +60 -0
- data/lib/pipedream.rb +18 -1
- data/lib/template/.pipedream/pipeline.rb.tt +40 -0
- data/lib/template/.pipedream/schedule.rb +3 -0
- data/lib/template/.pipedream/settings.yml +9 -0
- data/lib/template/.pipedream/sns.rb +14 -0
- data/pipedream.gemspec +25 -14
- data/vendor/aws_data/CHANGELOG.md +7 -0
- data/vendor/aws_data/Gemfile +4 -0
- data/vendor/aws_data/Gemfile.lock +48 -0
- data/vendor/aws_data/LICENSE.txt +21 -0
- data/vendor/aws_data/README.md +42 -0
- data/vendor/aws_data/Rakefile +6 -0
- data/vendor/aws_data/aws_data.gemspec +30 -0
- data/{bin → vendor/aws_data/bin}/console +1 -1
- data/{bin → vendor/aws_data/bin}/setup +0 -0
- data/vendor/aws_data/lib/aws_data/version.rb +3 -0
- data/vendor/aws_data/lib/aws_data.rb +91 -0
- data/vendor/aws_data/spec/aws_data_spec.rb +5 -0
- data/vendor/aws_data/spec/spec_helper.rb +14 -0
- data/vendor/cfn-status/Gemfile +4 -0
- data/vendor/cfn-status/Gemfile.lock +49 -0
- data/vendor/cfn-status/LICENSE.txt +21 -0
- data/vendor/cfn-status/README.md +56 -0
- data/vendor/cfn-status/Rakefile +6 -0
- data/vendor/cfn-status/bin/console +14 -0
- data/vendor/cfn-status/bin/setup +8 -0
- data/vendor/cfn-status/cfn-status.gemspec +30 -0
- data/vendor/cfn-status/lib/cfn/aws_service.rb +56 -0
- data/vendor/cfn-status/lib/cfn/status/version.rb +5 -0
- data/vendor/cfn-status/lib/cfn/status.rb +220 -0
- data/vendor/cfn-status/spec/cfn/status_spec.rb +81 -0
- data/vendor/cfn-status/spec/fixtures/cfn/stack-events-complete.json +1080 -0
- data/vendor/cfn-status/spec/fixtures/cfn/stack-events-in-progress.json +1080 -0
- data/vendor/cfn-status/spec/fixtures/cfn/stack-events-update-rollback-complete.json +1086 -0
- data/vendor/cfn-status/spec/spec_helper.rb +14 -0
- data/vendor/cfn_camelizer/CHANGELOG.md +10 -0
- data/vendor/cfn_camelizer/Gemfile +4 -0
- data/vendor/cfn_camelizer/LICENSE.txt +21 -0
- data/vendor/cfn_camelizer/README.md +40 -0
- data/vendor/cfn_camelizer/Rakefile +6 -0
- data/vendor/cfn_camelizer/bin/console +14 -0
- data/vendor/cfn_camelizer/bin/setup +8 -0
- data/vendor/cfn_camelizer/cfn_camelizer.gemspec +32 -0
- data/vendor/cfn_camelizer/lib/camelizer.yml +27 -0
- data/vendor/cfn_camelizer/lib/cfn_camelizer/version.rb +3 -0
- data/vendor/cfn_camelizer/lib/cfn_camelizer.rb +92 -0
- data/vendor/cfn_camelizer/spec/cfn_camelizer_spec.rb +79 -0
- data/vendor/cfn_camelizer/spec/spec_helper.rb +14 -0
- metadata +295 -23
- data/.travis.yml +0 -7
@@ -0,0 +1,220 @@
|
|
1
|
+
require "cfn/status/version"
|
2
|
+
|
3
|
+
module Cfn
|
4
|
+
autoload :AwsService, "cfn/aws_service"
|
5
|
+
|
6
|
+
class Status
|
7
|
+
class Error < StandardError; end
|
8
|
+
|
9
|
+
include AwsService
|
10
|
+
|
11
|
+
attr_reader :events
|
12
|
+
def initialize(stack_name, options={})
|
13
|
+
@stack_name = stack_name
|
14
|
+
@options = options
|
15
|
+
reset
|
16
|
+
end
|
17
|
+
|
18
|
+
# used for the bolt cfn status command
|
19
|
+
def run
|
20
|
+
unless stack_exists?(@stack_name)
|
21
|
+
puts "The stack #{@stack_name.color(:green)} does not exist."
|
22
|
+
return true
|
23
|
+
end
|
24
|
+
|
25
|
+
resp = cfn.describe_stacks(stack_name: @stack_name)
|
26
|
+
stack = resp.stacks.first
|
27
|
+
|
28
|
+
puts "The current status for the stack #{@stack_name.color(:green)} is #{stack.stack_status.color(:green)}"
|
29
|
+
if stack.stack_status =~ /_IN_PROGRESS$/
|
30
|
+
puts "Stack events (tailing):"
|
31
|
+
# tail all events until done
|
32
|
+
@hide_time_took = true
|
33
|
+
wait
|
34
|
+
else
|
35
|
+
puts "Stack events:"
|
36
|
+
# show the last events that was user initiated
|
37
|
+
refresh_events
|
38
|
+
show_events(true)
|
39
|
+
end
|
40
|
+
success?
|
41
|
+
end
|
42
|
+
|
43
|
+
def reset
|
44
|
+
@events = [] # constantly replaced with recent events
|
45
|
+
@last_shown_event_id = nil
|
46
|
+
@stack_deletion_completed = nil
|
47
|
+
end
|
48
|
+
|
49
|
+
# check for /(_COMPLETE|_FAILED)$/ status
|
50
|
+
def wait
|
51
|
+
puts "Waiting for stack to complete"
|
52
|
+
start_time = Time.now
|
53
|
+
|
54
|
+
refresh_events
|
55
|
+
until completed || @stack_deletion_completed
|
56
|
+
show_events
|
57
|
+
end
|
58
|
+
show_events(true) # show the final event
|
59
|
+
|
60
|
+
if @stack_deletion_completed
|
61
|
+
puts "Stack #{@stack_name} deleted."
|
62
|
+
return
|
63
|
+
end
|
64
|
+
|
65
|
+
if last_event_status =~ /_FAILED/
|
66
|
+
puts "Stack failed: #{last_event_status}".color(:red)
|
67
|
+
puts "Stack reason #{@events[0]["resource_status_reason"]}".color(:red)
|
68
|
+
elsif last_event_status =~ /_ROLLBACK_/
|
69
|
+
puts "Stack rolled back: #{last_event_status}".color(:red)
|
70
|
+
else # success
|
71
|
+
puts "Stack success status: #{last_event_status}".color(:green)
|
72
|
+
end
|
73
|
+
|
74
|
+
# Never gets here when deleting a stack because the describe stack returns nothing
|
75
|
+
# once the stack is deleted. Gets here for stack create and update though.
|
76
|
+
return if @hide_time_took # set in run
|
77
|
+
took = Time.now - start_time
|
78
|
+
puts "Time took for stack deployment: #{pretty_time(took).color(:green)}."
|
79
|
+
success?
|
80
|
+
end
|
81
|
+
|
82
|
+
def completed
|
83
|
+
last_event_status =~ /(_COMPLETE|_FAILED)$/ &&
|
84
|
+
@events[0]["logical_resource_id"] == @stack_name &&
|
85
|
+
@events[0]["resource_type"] == "AWS::CloudFormation::Stack"
|
86
|
+
end
|
87
|
+
|
88
|
+
def last_event_status
|
89
|
+
@events[0]["resource_status"]
|
90
|
+
end
|
91
|
+
|
92
|
+
# Only shows new events
|
93
|
+
def show_events(final=false)
|
94
|
+
if @last_shown_event_id.nil?
|
95
|
+
i = find_index(:start, final)
|
96
|
+
print_events(i)
|
97
|
+
else
|
98
|
+
i = find_index(:last_shown, final)
|
99
|
+
# puts "last_shown index #{i}"
|
100
|
+
print_events(i-1) unless i == 0
|
101
|
+
end
|
102
|
+
|
103
|
+
return if final
|
104
|
+
sleep 5 unless ENV['TEST']
|
105
|
+
refresh_events
|
106
|
+
end
|
107
|
+
|
108
|
+
def print_events(i)
|
109
|
+
@events[0..i].reverse.each do |e|
|
110
|
+
print_event(e)
|
111
|
+
end
|
112
|
+
@last_shown_event_id = @events[0]["event_id"]
|
113
|
+
# puts "@last_shown_event_id #{@last_shown_event_id.inspect}"
|
114
|
+
end
|
115
|
+
|
116
|
+
def print_event(e)
|
117
|
+
message = [
|
118
|
+
event_time(e["timestamp"]),
|
119
|
+
e["resource_status"],
|
120
|
+
e["resource_type"],
|
121
|
+
e["logical_resource_id"],
|
122
|
+
e["resource_status_reason"]
|
123
|
+
].join(" ")
|
124
|
+
message = message.color(:red) if e["resource_status"] =~ /_FAILED/
|
125
|
+
puts message
|
126
|
+
end
|
127
|
+
|
128
|
+
# https://stackoverflow.com/questions/18000432/rails-12-hour-am-pm-range-for-a-day
|
129
|
+
def event_time(timestamp)
|
130
|
+
Time.parse(timestamp.to_s).localtime.strftime("%I:%M:%S%p")
|
131
|
+
end
|
132
|
+
|
133
|
+
# refreshes the loaded events in memory
|
134
|
+
def refresh_events
|
135
|
+
resp = cfn.describe_stack_events(stack_name: @stack_name)
|
136
|
+
@events = resp["stack_events"]
|
137
|
+
rescue Aws::CloudFormation::Errors::ValidationError => e
|
138
|
+
if e.message =~ /Stack .* does not exis/
|
139
|
+
@stack_deletion_completed = true
|
140
|
+
else
|
141
|
+
raise
|
142
|
+
end
|
143
|
+
end
|
144
|
+
|
145
|
+
def find_index(name, final=false)
|
146
|
+
send("#{name}_index", final)
|
147
|
+
end
|
148
|
+
|
149
|
+
def start_index(final=false)
|
150
|
+
index = @events.find_index do |event|
|
151
|
+
event["resource_type"] == "AWS::CloudFormation::Stack" &&
|
152
|
+
event["resource_status_reason"] == "User Initiated"
|
153
|
+
end
|
154
|
+
# Instead of paginating until until we find the first "User Initiated" "AWS::CloudFormation::Stack" event
|
155
|
+
# we'll use the max.
|
156
|
+
index ? index : @events.size - 1
|
157
|
+
end
|
158
|
+
|
159
|
+
def last_shown_index(_)
|
160
|
+
@events.find_index do |event|
|
161
|
+
event["event_id"] == @last_shown_event_id
|
162
|
+
end
|
163
|
+
end
|
164
|
+
|
165
|
+
def success?
|
166
|
+
resource_status = @events[0]["resource_status"]
|
167
|
+
%w[CREATE_COMPLETE UPDATE_COMPLETE].include?(resource_status)
|
168
|
+
end
|
169
|
+
|
170
|
+
def update_rollback?
|
171
|
+
@events[0]["resource_status"] == "UPDATE_ROLLBACK_COMPLETE"
|
172
|
+
end
|
173
|
+
|
174
|
+
def find_update_failed_event
|
175
|
+
i = @events.find_index do |event|
|
176
|
+
event["resource_type"] == "AWS::CloudFormation::Stack" &&
|
177
|
+
event["resource_status_reason"] == "User Initiated"
|
178
|
+
end
|
179
|
+
|
180
|
+
@events[0..i].reverse.find do |e|
|
181
|
+
e["resource_status"] == "UPDATE_FAILED"
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
def rollback_error_message
|
186
|
+
return unless update_rollback?
|
187
|
+
|
188
|
+
event = find_update_failed_event
|
189
|
+
return unless event
|
190
|
+
|
191
|
+
reason = event["resource_status_reason"]
|
192
|
+
messages_map.each do |pattern, message|
|
193
|
+
if reason =~ pattern
|
194
|
+
return message
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
reason # default message is original reason if not found in messages map
|
199
|
+
end
|
200
|
+
|
201
|
+
def messages_map
|
202
|
+
{
|
203
|
+
/CloudFormation cannot update a stack when a custom-named resource requires replacing/ => "A workaround is to run ufo again with STATIC_NAME=0 and to switch to dynamic names for resources. Then run ufo again with STATIC_NAME=1 to get back to statically name resources. Note, there are caveats with the workaround.",
|
204
|
+
/cannot be associated with more than one load balancer/ => "There's was an issue updating the stack. Target groups can only be associated with one load balancer at a time. The workaround for this is to use UFO_FORCE_TARGET_GROUP=1 and run the command again. This will force the recreation of the target group resource.",
|
205
|
+
/SetSubnets is not supported for load balancers of type/ => "Changing subnets for Network Load Balancers is currently not supported. You can try workarouding this with UFO_FORCE_ELB=1 and run the command again. This will force the recreation of the elb resource."
|
206
|
+
}
|
207
|
+
end
|
208
|
+
|
209
|
+
# http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
|
210
|
+
def pretty_time(total_seconds)
|
211
|
+
minutes = (total_seconds / 60) % 60
|
212
|
+
seconds = total_seconds % 60
|
213
|
+
if total_seconds < 60
|
214
|
+
"#{seconds.to_i}s"
|
215
|
+
else
|
216
|
+
"#{minutes.to_i}m #{seconds.to_i}s"
|
217
|
+
end
|
218
|
+
end
|
219
|
+
end
|
220
|
+
end
|
@@ -0,0 +1,81 @@
|
|
1
|
+
RSpec.describe Cfn::Status do
|
2
|
+
it "has a version number" do
|
3
|
+
expect(Cfn::Status::VERSION).not_to be nil
|
4
|
+
end
|
5
|
+
|
6
|
+
let(:status) do
|
7
|
+
status = Cfn::Status.new("test-stack")
|
8
|
+
allow(status).to receive(:cfn).and_return(cfn)
|
9
|
+
status
|
10
|
+
end
|
11
|
+
let(:cfn) do
|
12
|
+
service = double("service").as_null_object
|
13
|
+
allow(service).to receive(:describe_stacks).and_return(stack_status)
|
14
|
+
allow(service).to receive(:describe_stack_events).and_return(stack_events)
|
15
|
+
service
|
16
|
+
end
|
17
|
+
|
18
|
+
context "in progress" do
|
19
|
+
let(:stack_status) { "UPDATE_IN_PROGRESS" }
|
20
|
+
let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-in-progress.json")) }
|
21
|
+
it "lists events since user initiated event" do
|
22
|
+
status.refresh_events
|
23
|
+
i = status.find_index(:start)
|
24
|
+
expect(i).to eq 15
|
25
|
+
# uncomment to view and debug
|
26
|
+
# status.show_events
|
27
|
+
# puts "****"
|
28
|
+
# status.show_events # show not show anything
|
29
|
+
end
|
30
|
+
|
31
|
+
it "lists events since last shown event" do
|
32
|
+
# first display
|
33
|
+
status.refresh_events
|
34
|
+
i = status.find_index(:start)
|
35
|
+
expect(i).to eq 15
|
36
|
+
|
37
|
+
# move the last event back in time 4 events, so should print 3 events
|
38
|
+
status.instance_variable_set(:@last_shown_event_id, "TargetGroup-ec634c43-b887-4bde-a525-7c69782865a6")
|
39
|
+
|
40
|
+
captured_events = []
|
41
|
+
allow(status).to receive(:print_event) do |e|
|
42
|
+
captured_events << "#{e["resource_type"]} #{e["resource_status"]}"
|
43
|
+
end
|
44
|
+
status.show_events
|
45
|
+
expect(captured_events).to eq([
|
46
|
+
"AWS::ElasticLoadBalancingV2::LoadBalancer DELETE_IN_PROGRESS",
|
47
|
+
"AWS::ElasticLoadBalancingV2::LoadBalancer DELETE_COMPLETE",
|
48
|
+
"AWS::EC2::SecurityGroup DELETE_IN_PROGRESS",
|
49
|
+
])
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
context "complete" do
|
54
|
+
let(:stack_status) { "UPDATE_COMPLETE" }
|
55
|
+
let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-complete.json")) }
|
56
|
+
it "lists events all the way to completion" do
|
57
|
+
status.refresh_events
|
58
|
+
i = status.find_index(:start)
|
59
|
+
expect(i).to eq 17
|
60
|
+
# uncomment to view and debug
|
61
|
+
# status.show_events
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
context "update_rollback" do
|
66
|
+
let(:stack_status) { "UPDATE_ROLLBACK_COMPLETE" }
|
67
|
+
let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-update-rollback-complete.json")) }
|
68
|
+
it "lists events all the way to update rollback complete" do
|
69
|
+
status.refresh_events
|
70
|
+
expect(status.success?).to be false
|
71
|
+
expect(status.update_rollback?).to be true
|
72
|
+
expect(status.rollback_error_message).to include("STATIC_NAME")
|
73
|
+
|
74
|
+
# i = status.find_index(:start)
|
75
|
+
# expect(i).to eq 17
|
76
|
+
# uncomment to view and debug
|
77
|
+
# status.show_events
|
78
|
+
end
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|