pipedream 0.1.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (106) hide show
  1. checksums.yaml +4 -4
  2. data/.cody/buildspec.yml +9 -0
  3. data/.cody/project.rb +4 -0
  4. data/.gitignore +16 -11
  5. data/.gitmodules +9 -0
  6. data/.rspec +1 -1
  7. data/CHANGELOG.md +34 -0
  8. data/Gemfile +3 -1
  9. data/Gemfile.lock +112 -0
  10. data/Guardfile +19 -0
  11. data/LICENSE.txt +18 -17
  12. data/README.md +55 -21
  13. data/Rakefile +10 -2
  14. data/exe/pipe +14 -0
  15. data/exe/pipedream +14 -0
  16. data/lib/pipedream/autoloader.rb +21 -0
  17. data/lib/pipedream/aws_services/helpers.rb +71 -0
  18. data/lib/pipedream/aws_services.rb +20 -0
  19. data/lib/pipedream/build.rb +13 -0
  20. data/lib/pipedream/cli.rb +60 -0
  21. data/lib/pipedream/command.rb +82 -0
  22. data/lib/pipedream/completer/script.rb +6 -0
  23. data/lib/pipedream/completer/script.sh +10 -0
  24. data/lib/pipedream/completer.rb +159 -0
  25. data/lib/pipedream/core.rb +63 -0
  26. data/lib/pipedream/create.rb +12 -0
  27. data/lib/pipedream/delete.rb +27 -0
  28. data/lib/pipedream/deploy.rb +40 -0
  29. data/lib/pipedream/dsl/pipeline/approve.rb +34 -0
  30. data/lib/pipedream/dsl/pipeline/codebuild.rb +57 -0
  31. data/lib/pipedream/dsl/pipeline/github.rb +42 -0
  32. data/lib/pipedream/dsl/pipeline.rb +37 -0
  33. data/lib/pipedream/dsl/role.rb +50 -0
  34. data/lib/pipedream/dsl/schedule.rb +30 -0
  35. data/lib/pipedream/dsl/sns.rb +15 -0
  36. data/lib/pipedream/dsl/ssm.rb +22 -0
  37. data/lib/pipedream/dsl/webhook.rb +27 -0
  38. data/lib/pipedream/evaluate.rb +47 -0
  39. data/lib/pipedream/help/completion.md +22 -0
  40. data/lib/pipedream/help/completion_script.md +3 -0
  41. data/lib/pipedream/help/deploy.md +54 -0
  42. data/lib/pipedream/help/start.md +20 -0
  43. data/lib/pipedream/help.rb +9 -0
  44. data/lib/pipedream/init.rb +68 -0
  45. data/lib/pipedream/pipeline/s3_bucket.rb +88 -0
  46. data/lib/pipedream/pipeline.rb +61 -0
  47. data/lib/pipedream/role.rb +181 -0
  48. data/lib/pipedream/schedule.rb +99 -0
  49. data/lib/pipedream/sequence.rb +66 -0
  50. data/lib/pipedream/setting.rb +82 -0
  51. data/lib/pipedream/sns.rb +43 -0
  52. data/lib/pipedream/stack.rb +95 -0
  53. data/lib/pipedream/start.rb +84 -0
  54. data/lib/pipedream/update.rb +12 -0
  55. data/lib/pipedream/version.rb +1 -1
  56. data/lib/pipedream/webhook.rb +60 -0
  57. data/lib/pipedream.rb +18 -1
  58. data/lib/template/.pipedream/pipeline.rb.tt +40 -0
  59. data/lib/template/.pipedream/schedule.rb +3 -0
  60. data/lib/template/.pipedream/settings.yml +9 -0
  61. data/lib/template/.pipedream/sns.rb +14 -0
  62. data/pipedream.gemspec +25 -14
  63. data/vendor/aws_data/CHANGELOG.md +7 -0
  64. data/vendor/aws_data/Gemfile +4 -0
  65. data/vendor/aws_data/Gemfile.lock +48 -0
  66. data/vendor/aws_data/LICENSE.txt +21 -0
  67. data/vendor/aws_data/README.md +42 -0
  68. data/vendor/aws_data/Rakefile +6 -0
  69. data/vendor/aws_data/aws_data.gemspec +30 -0
  70. data/{bin → vendor/aws_data/bin}/console +1 -1
  71. data/{bin → vendor/aws_data/bin}/setup +0 -0
  72. data/vendor/aws_data/lib/aws_data/version.rb +3 -0
  73. data/vendor/aws_data/lib/aws_data.rb +91 -0
  74. data/vendor/aws_data/spec/aws_data_spec.rb +5 -0
  75. data/vendor/aws_data/spec/spec_helper.rb +14 -0
  76. data/vendor/cfn-status/Gemfile +4 -0
  77. data/vendor/cfn-status/Gemfile.lock +49 -0
  78. data/vendor/cfn-status/LICENSE.txt +21 -0
  79. data/vendor/cfn-status/README.md +56 -0
  80. data/vendor/cfn-status/Rakefile +6 -0
  81. data/vendor/cfn-status/bin/console +14 -0
  82. data/vendor/cfn-status/bin/setup +8 -0
  83. data/vendor/cfn-status/cfn-status.gemspec +30 -0
  84. data/vendor/cfn-status/lib/cfn/aws_service.rb +56 -0
  85. data/vendor/cfn-status/lib/cfn/status/version.rb +5 -0
  86. data/vendor/cfn-status/lib/cfn/status.rb +220 -0
  87. data/vendor/cfn-status/spec/cfn/status_spec.rb +81 -0
  88. data/vendor/cfn-status/spec/fixtures/cfn/stack-events-complete.json +1080 -0
  89. data/vendor/cfn-status/spec/fixtures/cfn/stack-events-in-progress.json +1080 -0
  90. data/vendor/cfn-status/spec/fixtures/cfn/stack-events-update-rollback-complete.json +1086 -0
  91. data/vendor/cfn-status/spec/spec_helper.rb +14 -0
  92. data/vendor/cfn_camelizer/CHANGELOG.md +10 -0
  93. data/vendor/cfn_camelizer/Gemfile +4 -0
  94. data/vendor/cfn_camelizer/LICENSE.txt +21 -0
  95. data/vendor/cfn_camelizer/README.md +40 -0
  96. data/vendor/cfn_camelizer/Rakefile +6 -0
  97. data/vendor/cfn_camelizer/bin/console +14 -0
  98. data/vendor/cfn_camelizer/bin/setup +8 -0
  99. data/vendor/cfn_camelizer/cfn_camelizer.gemspec +32 -0
  100. data/vendor/cfn_camelizer/lib/camelizer.yml +27 -0
  101. data/vendor/cfn_camelizer/lib/cfn_camelizer/version.rb +3 -0
  102. data/vendor/cfn_camelizer/lib/cfn_camelizer.rb +92 -0
  103. data/vendor/cfn_camelizer/spec/cfn_camelizer_spec.rb +79 -0
  104. data/vendor/cfn_camelizer/spec/spec_helper.rb +14 -0
  105. metadata +295 -23
  106. data/.travis.yml +0 -7
@@ -0,0 +1,220 @@
1
+ require "cfn/status/version"
2
+
3
+ module Cfn
4
+ autoload :AwsService, "cfn/aws_service"
5
+
6
+ class Status
7
+ class Error < StandardError; end
8
+
9
+ include AwsService
10
+
11
+ attr_reader :events
12
+ def initialize(stack_name, options={})
13
+ @stack_name = stack_name
14
+ @options = options
15
+ reset
16
+ end
17
+
18
+ # used for the bolt cfn status command
19
+ def run
20
+ unless stack_exists?(@stack_name)
21
+ puts "The stack #{@stack_name.color(:green)} does not exist."
22
+ return true
23
+ end
24
+
25
+ resp = cfn.describe_stacks(stack_name: @stack_name)
26
+ stack = resp.stacks.first
27
+
28
+ puts "The current status for the stack #{@stack_name.color(:green)} is #{stack.stack_status.color(:green)}"
29
+ if stack.stack_status =~ /_IN_PROGRESS$/
30
+ puts "Stack events (tailing):"
31
+ # tail all events until done
32
+ @hide_time_took = true
33
+ wait
34
+ else
35
+ puts "Stack events:"
36
+ # show the last events that was user initiated
37
+ refresh_events
38
+ show_events(true)
39
+ end
40
+ success?
41
+ end
42
+
43
+ def reset
44
+ @events = [] # constantly replaced with recent events
45
+ @last_shown_event_id = nil
46
+ @stack_deletion_completed = nil
47
+ end
48
+
49
+ # check for /(_COMPLETE|_FAILED)$/ status
50
+ def wait
51
+ puts "Waiting for stack to complete"
52
+ start_time = Time.now
53
+
54
+ refresh_events
55
+ until completed || @stack_deletion_completed
56
+ show_events
57
+ end
58
+ show_events(true) # show the final event
59
+
60
+ if @stack_deletion_completed
61
+ puts "Stack #{@stack_name} deleted."
62
+ return
63
+ end
64
+
65
+ if last_event_status =~ /_FAILED/
66
+ puts "Stack failed: #{last_event_status}".color(:red)
67
+ puts "Stack reason #{@events[0]["resource_status_reason"]}".color(:red)
68
+ elsif last_event_status =~ /_ROLLBACK_/
69
+ puts "Stack rolled back: #{last_event_status}".color(:red)
70
+ else # success
71
+ puts "Stack success status: #{last_event_status}".color(:green)
72
+ end
73
+
74
+ # Never gets here when deleting a stack because the describe stack returns nothing
75
+ # once the stack is deleted. Gets here for stack create and update though.
76
+ return if @hide_time_took # set in run
77
+ took = Time.now - start_time
78
+ puts "Time took for stack deployment: #{pretty_time(took).color(:green)}."
79
+ success?
80
+ end
81
+
82
+ def completed
83
+ last_event_status =~ /(_COMPLETE|_FAILED)$/ &&
84
+ @events[0]["logical_resource_id"] == @stack_name &&
85
+ @events[0]["resource_type"] == "AWS::CloudFormation::Stack"
86
+ end
87
+
88
+ def last_event_status
89
+ @events[0]["resource_status"]
90
+ end
91
+
92
+ # Only shows new events
93
+ def show_events(final=false)
94
+ if @last_shown_event_id.nil?
95
+ i = find_index(:start, final)
96
+ print_events(i)
97
+ else
98
+ i = find_index(:last_shown, final)
99
+ # puts "last_shown index #{i}"
100
+ print_events(i-1) unless i == 0
101
+ end
102
+
103
+ return if final
104
+ sleep 5 unless ENV['TEST']
105
+ refresh_events
106
+ end
107
+
108
+ def print_events(i)
109
+ @events[0..i].reverse.each do |e|
110
+ print_event(e)
111
+ end
112
+ @last_shown_event_id = @events[0]["event_id"]
113
+ # puts "@last_shown_event_id #{@last_shown_event_id.inspect}"
114
+ end
115
+
116
+ def print_event(e)
117
+ message = [
118
+ event_time(e["timestamp"]),
119
+ e["resource_status"],
120
+ e["resource_type"],
121
+ e["logical_resource_id"],
122
+ e["resource_status_reason"]
123
+ ].join(" ")
124
+ message = message.color(:red) if e["resource_status"] =~ /_FAILED/
125
+ puts message
126
+ end
127
+
128
+ # https://stackoverflow.com/questions/18000432/rails-12-hour-am-pm-range-for-a-day
129
+ def event_time(timestamp)
130
+ Time.parse(timestamp.to_s).localtime.strftime("%I:%M:%S%p")
131
+ end
132
+
133
+ # refreshes the loaded events in memory
134
+ def refresh_events
135
+ resp = cfn.describe_stack_events(stack_name: @stack_name)
136
+ @events = resp["stack_events"]
137
+ rescue Aws::CloudFormation::Errors::ValidationError => e
138
+ if e.message =~ /Stack .* does not exis/
139
+ @stack_deletion_completed = true
140
+ else
141
+ raise
142
+ end
143
+ end
144
+
145
+ def find_index(name, final=false)
146
+ send("#{name}_index", final)
147
+ end
148
+
149
+ def start_index(final=false)
150
+ index = @events.find_index do |event|
151
+ event["resource_type"] == "AWS::CloudFormation::Stack" &&
152
+ event["resource_status_reason"] == "User Initiated"
153
+ end
154
+ # Instead of paginating until until we find the first "User Initiated" "AWS::CloudFormation::Stack" event
155
+ # we'll use the max.
156
+ index ? index : @events.size - 1
157
+ end
158
+
159
+ def last_shown_index(_)
160
+ @events.find_index do |event|
161
+ event["event_id"] == @last_shown_event_id
162
+ end
163
+ end
164
+
165
+ def success?
166
+ resource_status = @events[0]["resource_status"]
167
+ %w[CREATE_COMPLETE UPDATE_COMPLETE].include?(resource_status)
168
+ end
169
+
170
+ def update_rollback?
171
+ @events[0]["resource_status"] == "UPDATE_ROLLBACK_COMPLETE"
172
+ end
173
+
174
+ def find_update_failed_event
175
+ i = @events.find_index do |event|
176
+ event["resource_type"] == "AWS::CloudFormation::Stack" &&
177
+ event["resource_status_reason"] == "User Initiated"
178
+ end
179
+
180
+ @events[0..i].reverse.find do |e|
181
+ e["resource_status"] == "UPDATE_FAILED"
182
+ end
183
+ end
184
+
185
+ def rollback_error_message
186
+ return unless update_rollback?
187
+
188
+ event = find_update_failed_event
189
+ return unless event
190
+
191
+ reason = event["resource_status_reason"]
192
+ messages_map.each do |pattern, message|
193
+ if reason =~ pattern
194
+ return message
195
+ end
196
+ end
197
+
198
+ reason # default message is original reason if not found in messages map
199
+ end
200
+
201
+ def messages_map
202
+ {
203
+ /CloudFormation cannot update a stack when a custom-named resource requires replacing/ => "A workaround is to run ufo again with STATIC_NAME=0 and to switch to dynamic names for resources. Then run ufo again with STATIC_NAME=1 to get back to statically name resources. Note, there are caveats with the workaround.",
204
+ /cannot be associated with more than one load balancer/ => "There's was an issue updating the stack. Target groups can only be associated with one load balancer at a time. The workaround for this is to use UFO_FORCE_TARGET_GROUP=1 and run the command again. This will force the recreation of the target group resource.",
205
+ /SetSubnets is not supported for load balancers of type/ => "Changing subnets for Network Load Balancers is currently not supported. You can try workarouding this with UFO_FORCE_ELB=1 and run the command again. This will force the recreation of the elb resource."
206
+ }
207
+ end
208
+
209
+ # http://stackoverflow.com/questions/4175733/convert-duration-to-hoursminutesseconds-or-similar-in-rails-3-or-ruby
210
+ def pretty_time(total_seconds)
211
+ minutes = (total_seconds / 60) % 60
212
+ seconds = total_seconds % 60
213
+ if total_seconds < 60
214
+ "#{seconds.to_i}s"
215
+ else
216
+ "#{minutes.to_i}m #{seconds.to_i}s"
217
+ end
218
+ end
219
+ end
220
+ end
@@ -0,0 +1,81 @@
1
+ RSpec.describe Cfn::Status do
2
+ it "has a version number" do
3
+ expect(Cfn::Status::VERSION).not_to be nil
4
+ end
5
+
6
+ let(:status) do
7
+ status = Cfn::Status.new("test-stack")
8
+ allow(status).to receive(:cfn).and_return(cfn)
9
+ status
10
+ end
11
+ let(:cfn) do
12
+ service = double("service").as_null_object
13
+ allow(service).to receive(:describe_stacks).and_return(stack_status)
14
+ allow(service).to receive(:describe_stack_events).and_return(stack_events)
15
+ service
16
+ end
17
+
18
+ context "in progress" do
19
+ let(:stack_status) { "UPDATE_IN_PROGRESS" }
20
+ let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-in-progress.json")) }
21
+ it "lists events since user initiated event" do
22
+ status.refresh_events
23
+ i = status.find_index(:start)
24
+ expect(i).to eq 15
25
+ # uncomment to view and debug
26
+ # status.show_events
27
+ # puts "****"
28
+ # status.show_events # show not show anything
29
+ end
30
+
31
+ it "lists events since last shown event" do
32
+ # first display
33
+ status.refresh_events
34
+ i = status.find_index(:start)
35
+ expect(i).to eq 15
36
+
37
+ # move the last event back in time 4 events, so should print 3 events
38
+ status.instance_variable_set(:@last_shown_event_id, "TargetGroup-ec634c43-b887-4bde-a525-7c69782865a6")
39
+
40
+ captured_events = []
41
+ allow(status).to receive(:print_event) do |e|
42
+ captured_events << "#{e["resource_type"]} #{e["resource_status"]}"
43
+ end
44
+ status.show_events
45
+ expect(captured_events).to eq([
46
+ "AWS::ElasticLoadBalancingV2::LoadBalancer DELETE_IN_PROGRESS",
47
+ "AWS::ElasticLoadBalancingV2::LoadBalancer DELETE_COMPLETE",
48
+ "AWS::EC2::SecurityGroup DELETE_IN_PROGRESS",
49
+ ])
50
+ end
51
+ end
52
+
53
+ context "complete" do
54
+ let(:stack_status) { "UPDATE_COMPLETE" }
55
+ let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-complete.json")) }
56
+ it "lists events all the way to completion" do
57
+ status.refresh_events
58
+ i = status.find_index(:start)
59
+ expect(i).to eq 17
60
+ # uncomment to view and debug
61
+ # status.show_events
62
+ end
63
+ end
64
+
65
+ context "update_rollback" do
66
+ let(:stack_status) { "UPDATE_ROLLBACK_COMPLETE" }
67
+ let(:stack_events) { JSON.load(IO.read("spec/fixtures/cfn/stack-events-update-rollback-complete.json")) }
68
+ it "lists events all the way to update rollback complete" do
69
+ status.refresh_events
70
+ expect(status.success?).to be false
71
+ expect(status.update_rollback?).to be true
72
+ expect(status.rollback_error_message).to include("STATIC_NAME")
73
+
74
+ # i = status.find_index(:start)
75
+ # expect(i).to eq 17
76
+ # uncomment to view and debug
77
+ # status.show_events
78
+ end
79
+ end
80
+ end
81
+