cloud_powers 0.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +9 -0
- data/.rspec +2 -0
- data/.travis.yml +5 -0
- data/Gemfile +4 -0
- data/Gemfile.lock +35 -0
- data/LICENSE.txt +21 -0
- data/README.md +41 -0
- data/Rakefile +6 -0
- data/bin/console +14 -0
- data/bin/setup +8 -0
- data/cloud_powers.gemspec +33 -0
- data/lib/cloud_powers.rb +20 -0
- data/lib/cloud_powers/auth.rb +15 -0
- data/lib/cloud_powers/aws_resources.rb +26 -0
- data/lib/cloud_powers/delegator.rb +37 -0
- data/lib/cloud_powers/helper.rb +121 -0
- data/lib/cloud_powers/self_awareness.rb +133 -0
- data/lib/cloud_powers/smash_error.rb +33 -0
- data/lib/cloud_powers/storage.rb +44 -0
- data/lib/cloud_powers/synapse/pipe.rb +97 -0
- data/lib/cloud_powers/synapse/queue.rb +112 -0
- data/lib/cloud_powers/synapse/synapse.rb +12 -0
- data/lib/cloud_powers/version.rb +3 -0
- data/lib/cloud_powers/workflow.rb +32 -0
- metadata +112 -0
checksums.yaml
ADDED
@@ -0,0 +1,7 @@
|
|
1
|
+
---
|
2
|
+
SHA1:
|
3
|
+
metadata.gz: 52e44795ffd76ed0a74e12a312f7602c908f95a4
|
4
|
+
data.tar.gz: 31eda48bf52bfe9fc3e36048e979db2bea25d56f
|
5
|
+
SHA512:
|
6
|
+
metadata.gz: acb3cf5189b2384c691792a2fdf865cae9345a964ca0f51dac06d0a78d9427b73648d0592891a8a743b36df4cd483c7fcda204d9fedb5debb3c1a8697014b1fc
|
7
|
+
data.tar.gz: 4f8a188451bc4126af3ed04b61ffba0c6852df7dff41c365033435ce26958a7aaf7e9c9ed4aca803ac23c249242f42bd9f9cc7d20d55fc5d2659b409179c2ec2
|
data/.gitignore
ADDED
data/.rspec
ADDED
data/.travis.yml
ADDED
data/Gemfile
ADDED
data/Gemfile.lock
ADDED
@@ -0,0 +1,35 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
cloud_powers (0.1.0)
|
5
|
+
|
6
|
+
GEM
|
7
|
+
remote: https://rubygems.org/
|
8
|
+
specs:
|
9
|
+
diff-lcs (1.2.5)
|
10
|
+
rake (10.4.2)
|
11
|
+
rspec (3.5.0)
|
12
|
+
rspec-core (~> 3.5.0)
|
13
|
+
rspec-expectations (~> 3.5.0)
|
14
|
+
rspec-mocks (~> 3.5.0)
|
15
|
+
rspec-core (3.5.3)
|
16
|
+
rspec-support (~> 3.5.0)
|
17
|
+
rspec-expectations (3.5.0)
|
18
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
19
|
+
rspec-support (~> 3.5.0)
|
20
|
+
rspec-mocks (3.5.0)
|
21
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
22
|
+
rspec-support (~> 3.5.0)
|
23
|
+
rspec-support (3.5.0)
|
24
|
+
|
25
|
+
PLATFORMS
|
26
|
+
ruby
|
27
|
+
|
28
|
+
DEPENDENCIES
|
29
|
+
bundler (~> 1.12)
|
30
|
+
cloud_powers!
|
31
|
+
rake (~> 10.0)
|
32
|
+
rspec (~> 3.0)
|
33
|
+
|
34
|
+
BUNDLED WITH
|
35
|
+
1.12.5
|
data/LICENSE.txt
ADDED
@@ -0,0 +1,21 @@
|
|
1
|
+
The MIT License (MIT)
|
2
|
+
|
3
|
+
Copyright (c) 2016 Adam Phillipps
|
4
|
+
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
7
|
+
in the Software without restriction, including without limitation the rights
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
10
|
+
furnished to do so, subject to the following conditions:
|
11
|
+
|
12
|
+
The above copyright notice and this permission notice shall be included in
|
13
|
+
all copies or substantial portions of the Software.
|
14
|
+
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
|
21
|
+
THE SOFTWARE.
|
data/README.md
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
# CloudPowers
|
2
|
+
|
3
|
+
Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/cloud_powers`. To experiment with that code, run `bin/console` for an interactive prompt.
|
4
|
+
|
5
|
+
TODO: Delete this and the text above, and describe your gem
|
6
|
+
|
7
|
+
## Installation
|
8
|
+
|
9
|
+
Add this line to your application's Gemfile:
|
10
|
+
|
11
|
+
```ruby
|
12
|
+
gem 'cloud_powers'
|
13
|
+
```
|
14
|
+
|
15
|
+
And then execute:
|
16
|
+
|
17
|
+
$ bundle
|
18
|
+
|
19
|
+
Or install it yourself as:
|
20
|
+
|
21
|
+
$ gem install cloud_powers
|
22
|
+
|
23
|
+
## Usage
|
24
|
+
|
25
|
+
TODO: Write usage instructions here
|
26
|
+
|
27
|
+
## Development
|
28
|
+
|
29
|
+
After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
|
30
|
+
|
31
|
+
To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
|
32
|
+
|
33
|
+
## Contributing
|
34
|
+
|
35
|
+
Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/cloud_powers.
|
36
|
+
|
37
|
+
|
38
|
+
## License
|
39
|
+
|
40
|
+
The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
|
41
|
+
|
data/Rakefile
ADDED
data/bin/console
ADDED
@@ -0,0 +1,14 @@
|
|
1
|
+
#!/usr/bin/env ruby
|
2
|
+
|
3
|
+
require "bundler/setup"
|
4
|
+
require "cloud_powers"
|
5
|
+
|
6
|
+
# You can add fixtures and/or initialization code here to make experimenting
|
7
|
+
# with your gem easier. You can also use a different console, if you like.
|
8
|
+
|
9
|
+
# (If you use this, don't forget to add pry to your Gemfile!)
|
10
|
+
# require "pry"
|
11
|
+
# Pry.start
|
12
|
+
|
13
|
+
require "irb"
|
14
|
+
IRB.start
|
data/bin/setup
ADDED
@@ -0,0 +1,33 @@
|
|
1
|
+
# coding: utf-8
|
2
|
+
lib = File.expand_path('../lib', __FILE__)
|
3
|
+
$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
|
4
|
+
require 'cloud_powers/version'
|
5
|
+
|
6
|
+
Gem::Specification.new do |spec|
|
7
|
+
spec.name = "cloud_powers"
|
8
|
+
spec.version = CloudPowers::VERSION
|
9
|
+
spec.authors = ["Adam Phillipps"]
|
10
|
+
spec.email = ["adam.phillipps@gmail.com"]
|
11
|
+
|
12
|
+
spec.summary = %q{Cloud providers wrapper. Currently only AWS is supported.}
|
13
|
+
spec.description = %q{CloudPowers is a wrapper around AWS and other cloud services. This wrapper was developed specifically for the Brain project but can be used in any other ruby project.}
|
14
|
+
spec.homepage = "https://github.com/adam-phillipps/cloud_powers"
|
15
|
+
spec.license = "MIT"
|
16
|
+
|
17
|
+
# Prevent pushing this gem to RubyGems.org. To allow pushes either set the 'allowed_push_host'
|
18
|
+
# to allow pushing to a single host or delete this section to allow pushing to any host.
|
19
|
+
# if spec.respond_to?(:metadata)
|
20
|
+
# spec.metadata['allowed_push_host'] = "TODO: Set to 'http://mygemserver.com'"
|
21
|
+
# else
|
22
|
+
# raise "RubyGems 2.0 or newer is required to protect against public gem pushes."
|
23
|
+
# end
|
24
|
+
|
25
|
+
spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
|
26
|
+
spec.bindir = "exe"
|
27
|
+
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
28
|
+
spec.require_paths = ["lib"]
|
29
|
+
|
30
|
+
spec.add_development_dependency "bundler", "~> 1.12"
|
31
|
+
spec.add_development_dependency "rake", "~> 10.0"
|
32
|
+
spec.add_development_dependency "rspec", "~> 3.0"
|
33
|
+
end
|
data/lib/cloud_powers.rb
ADDED
@@ -0,0 +1,20 @@
|
|
1
|
+
require 'cloud_powers/version'
|
2
|
+
require 'cloud_powers/auth'
|
3
|
+
require 'cloud_powers/aws_resources'
|
4
|
+
require 'cloud_powers/delegator'
|
5
|
+
require 'cloud_powers/helper'
|
6
|
+
require 'cloud_powers/self_awareness'
|
7
|
+
require 'cloud_powers/storage'
|
8
|
+
require 'cloud_powers/workflow'
|
9
|
+
|
10
|
+
module Smash
|
11
|
+
module CloudPowers
|
12
|
+
extend Smash::CloudPowers::Auth
|
13
|
+
extend Smash::CloudPowers::Delegator
|
14
|
+
include Smash::CloudPowers::AwsResources
|
15
|
+
include Smash::CloudPowers::Helper
|
16
|
+
include Smash::CloudPowers::SelfAwareness
|
17
|
+
include Smash::CloudPowers::Storage
|
18
|
+
include Smash::CloudPowers::Synapse
|
19
|
+
end
|
20
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
require_relative 'auth'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module AwsResources
|
6
|
+
extend Smash::CloudPowers::Auth
|
7
|
+
|
8
|
+
def region
|
9
|
+
env('Aws Region')
|
10
|
+
end
|
11
|
+
|
12
|
+
def ec2
|
13
|
+
@ec2 ||= Aws::EC2::Client.new(
|
14
|
+
region: region,
|
15
|
+
credentials: Auth.creds
|
16
|
+
)
|
17
|
+
end
|
18
|
+
|
19
|
+
def image(name)
|
20
|
+
ec2.describe_images(
|
21
|
+
filters: [{ name: 'tag:aminame', values: [name.to_s] }]
|
22
|
+
).images.first
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require_relative 'auth'
|
2
|
+
require_relative 'aws_resources'
|
3
|
+
require_relative 'helper'
|
4
|
+
require_relative 'storage'
|
5
|
+
require 'byebug'
|
6
|
+
|
7
|
+
module Smash
|
8
|
+
module Delegator
|
9
|
+
extend Smash::CloudPowers::Auth
|
10
|
+
include Smash::CloudPowers::AwsResources
|
11
|
+
include Smash::CloudPowers::Helper
|
12
|
+
include Smash::CloudPowers::Storage
|
13
|
+
|
14
|
+
def build(id, msg)
|
15
|
+
body = JSON.parse(msg.body)
|
16
|
+
begin
|
17
|
+
task = body.delete('task')
|
18
|
+
if approved_task? task
|
19
|
+
source_task(task)
|
20
|
+
require_relative task_require_path(task)
|
21
|
+
Smash.const_get(to_pascal(task)).new(id, msg)
|
22
|
+
else
|
23
|
+
Task.new(id, msg) # returns a default Task
|
24
|
+
end
|
25
|
+
rescue JSON::ParserError => e
|
26
|
+
message = [msg.body, format_error_message(e)].join("\n")
|
27
|
+
logger.info "Message in backlog is ill-formatted: #{message}"
|
28
|
+
pipe_to(:status_stream) { sitrep(extraInfo: { message: message }) }
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def approved_task?(name = nil)
|
33
|
+
# TODO: improve this
|
34
|
+
['demo', 'testinz'].include? name
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,121 @@
|
|
1
|
+
require 'logger'
|
2
|
+
require 'pathname'
|
3
|
+
require 'syslog/logger'
|
4
|
+
require_relative 'smash_error'
|
5
|
+
require_relative 'auth'
|
6
|
+
require 'byebug'
|
7
|
+
|
8
|
+
module Smash
|
9
|
+
module CloudPowers
|
10
|
+
module Helper
|
11
|
+
include Smash::CloudPowers::Auth
|
12
|
+
|
13
|
+
def attr_map!(keys)
|
14
|
+
keys.map do |attr|
|
15
|
+
key = to_i_var(attr)
|
16
|
+
value = yield key if block_given?
|
17
|
+
instance_variable_set(key, value)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
|
21
|
+
def env(key)
|
22
|
+
ENV[to_snake(key).upcase]
|
23
|
+
end
|
24
|
+
|
25
|
+
def create_logger
|
26
|
+
logger = Logger.new(STDOUT)
|
27
|
+
logger.datetime_format = '%Y-%m-%d %H:%M:%S'
|
28
|
+
logger
|
29
|
+
end
|
30
|
+
|
31
|
+
def errors
|
32
|
+
# TODO: needs work
|
33
|
+
$errors ||= SmashError.instance
|
34
|
+
end
|
35
|
+
|
36
|
+
def format_error_message(error)
|
37
|
+
begin
|
38
|
+
[error.message, error.backtrace.join("\n")].join("\n")
|
39
|
+
rescue Exception => e
|
40
|
+
# if the formatting won't work, return the original exception
|
41
|
+
error
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def log_file
|
46
|
+
@log_file ||= env('LOG_FILE')
|
47
|
+
end
|
48
|
+
|
49
|
+
def logger
|
50
|
+
@logger ||= create_logger
|
51
|
+
end
|
52
|
+
|
53
|
+
# Sample usage:
|
54
|
+
# check_stuff = lambda { |params| return true }
|
55
|
+
# retry(3, check_stuff(params)) { do_stuff_that_needs_to_be_checked }
|
56
|
+
def retry(allowed_attempts = Float::Infinity, &test)
|
57
|
+
result = yield if block_given?
|
58
|
+
tries = 1
|
59
|
+
until test.call(result) || tries >= allowed_attempts
|
60
|
+
result = yield if block_given?
|
61
|
+
sleep 1
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
def task_path(file)
|
66
|
+
Pathname(__FILE__).parent.dirname + 'tasks' + to_ruby_file_name(file)
|
67
|
+
end
|
68
|
+
|
69
|
+
def task_require_path(file_name)
|
70
|
+
file = File.basename(file_name, File.extname(file_name))
|
71
|
+
Pathname(__FILE__).parent.dirname + 'tasks' + file
|
72
|
+
end
|
73
|
+
|
74
|
+
def to_camel(var)
|
75
|
+
var = var.to_s unless var.kind_of? String
|
76
|
+
step_one = to_snake(var)
|
77
|
+
step_two = to_pascal(step_one)
|
78
|
+
step_two[0, 1].downcase + step_two[1..-1]
|
79
|
+
end
|
80
|
+
|
81
|
+
def to_i_var(var)
|
82
|
+
"@#{to_snake(var)}"
|
83
|
+
end
|
84
|
+
|
85
|
+
def to_pascal(var)
|
86
|
+
var = var.to_s unless var.kind_of? String
|
87
|
+
var.gsub(/^(.{1})|\W.{1}|\_.{1}/) { |s| s.gsub(/[^a-z0-9]+/i, '').capitalize }
|
88
|
+
end
|
89
|
+
|
90
|
+
def to_ruby_file_name(name)
|
91
|
+
"#{to_snake(name)}.rb"
|
92
|
+
end
|
93
|
+
|
94
|
+
def to_snake(var)
|
95
|
+
var = var.to_s unless var.kind_of? String
|
96
|
+
var.gsub(/\W/, '_').downcase
|
97
|
+
end
|
98
|
+
|
99
|
+
def update_message_body(opts = {})
|
100
|
+
# TODO: find better implementation of merging nested hashes
|
101
|
+
# this should be fixed with Job #sitrep_message
|
102
|
+
# TODO: find a way to trim this method down and get rid
|
103
|
+
# of a lof of the repitition with these messages
|
104
|
+
# IDEA: throw events and have a separate thread listening. the separate
|
105
|
+
# thread could be a communication or status update thread
|
106
|
+
unless opts.kind_of? Hash
|
107
|
+
update = opts.to_s
|
108
|
+
opts[:extraInfo] = { message: update }
|
109
|
+
end
|
110
|
+
updated_extra_info = opts.delete(:extraInfo) || {}
|
111
|
+
|
112
|
+
{
|
113
|
+
instanceId: @instance_id || 'none-aquired',
|
114
|
+
type: 'status-update',
|
115
|
+
content: 'running',
|
116
|
+
extraInfo: updated_extra_info
|
117
|
+
}.merge(opts)
|
118
|
+
end
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
@@ -0,0 +1,133 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
Aws.use_bundled_cert!
|
3
|
+
require 'httparty'
|
4
|
+
require_relative 'helper'
|
5
|
+
require_relative './synapse/synapse'
|
6
|
+
|
7
|
+
module Smash
|
8
|
+
module CloudPowers
|
9
|
+
extend Smash::CloudPowers::Synapse::Pipe
|
10
|
+
extend Smash::CloudPowers::Synapse::Queue
|
11
|
+
|
12
|
+
module SelfAwareness
|
13
|
+
extend Smash::CloudPowers::Helper
|
14
|
+
extend Smash::CloudPowers::Synapse::Pipe
|
15
|
+
extend Smash::CloudPowers::Synapse::Queue
|
16
|
+
|
17
|
+
def boot_time
|
18
|
+
begin
|
19
|
+
@boot_time ||=
|
20
|
+
ec2.describe_instances(dry_run: env('testing'), instance_ids:[@instance_id]).
|
21
|
+
reservations[0].instances[0].launch_time.to_i
|
22
|
+
rescue Aws::EC2::Errors::DryRunOperation => e
|
23
|
+
logger.info "dry run for testing: #{format_error_message(e)}"
|
24
|
+
@boot_time ||= Time.now.to_i # comment the code below for development mode
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
def die!
|
29
|
+
Thread.kill(@status_thread) unless @status_thread.nil?
|
30
|
+
# blame = errors.sort_by(&:reverse).last.first
|
31
|
+
logger.info("The cause for the shutdown is TODO: fix SmashErrors")
|
32
|
+
|
33
|
+
pipe_to(:status_stream) do
|
34
|
+
{
|
35
|
+
instanceID: @instance_id,
|
36
|
+
type: 'status-update',
|
37
|
+
content: 'dying'
|
38
|
+
# extraInfo: blame
|
39
|
+
}
|
40
|
+
end
|
41
|
+
|
42
|
+
[:count, :wip].each do |queue|
|
43
|
+
delete_queue_message(queue, max_number_of_messages: 1)
|
44
|
+
end
|
45
|
+
send_logs_to_s3
|
46
|
+
begin
|
47
|
+
ec2.terminate_instances(dry_run: env('testing'), ids: [@instance_id])
|
48
|
+
rescue Aws::EC2::Error::DryRunOperation => e
|
49
|
+
logger.info "dry run testing in die! #{format_error_message(e)}"
|
50
|
+
@instance_id
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
def get_awareness!
|
55
|
+
keys = metadata_request
|
56
|
+
attr_map!(keys) { |key| metadata_request(key) }
|
57
|
+
boot_time # sets @boot_time
|
58
|
+
task_name # gets and sets @task_name
|
59
|
+
end
|
60
|
+
|
61
|
+
def task_name(ids = @instance_id)
|
62
|
+
# check tags for 'task'
|
63
|
+
if @task_name.nil?
|
64
|
+
resp = ec2.describe_instances(instance_ids: [ids].flatten)
|
65
|
+
@task_name = resp.reservations[0].instances[0].tags.select do |t|
|
66
|
+
t.value if t.key == 'taskType'
|
67
|
+
end
|
68
|
+
else
|
69
|
+
@task_name
|
70
|
+
end
|
71
|
+
end
|
72
|
+
|
73
|
+
def get_instance_url
|
74
|
+
if env('TESTING')
|
75
|
+
'https://test-url.com'
|
76
|
+
else
|
77
|
+
hostname_uri = 'http://169.254.169.254/latest/meta-data/public-hostname'
|
78
|
+
HTTParty.get(hostname_uri).parsed_response
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def metadata_request(key = '')
|
83
|
+
unless env('TESTING')
|
84
|
+
metadata_uri = "http://169.254.169.254/latest/meta-data/#{key}"
|
85
|
+
HTTParty.get(metadata_uri).parsed_response.split("\n")
|
86
|
+
else
|
87
|
+
@z ||= ['i-9254d106', 'ami-id', 'ami-launch-index', 'ami-manifest-path', 'network/thing']
|
88
|
+
if key == ''
|
89
|
+
@boogs = ['instance-id', 'ami-id', 'ami-launch-index', 'ami-manifest-path', 'network/interfaces/macs/mac/device-number']
|
90
|
+
else
|
91
|
+
@z.shift
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def run_time
|
97
|
+
# TODO: refactor to use valid time stamps for better tracking.
|
98
|
+
# reason -> separate regions or OSs etc.
|
99
|
+
Time.now.to_i - boot_time
|
100
|
+
end
|
101
|
+
|
102
|
+
def send_frequent_status_updates(opts = {})
|
103
|
+
sleep_time = opts.delete(:interval) || 10
|
104
|
+
stream = opts.delete(:stream_name)
|
105
|
+
while true
|
106
|
+
message = lambda { |o| update_message_body(o.merge(content: status)) }
|
107
|
+
logger.info "Send update to status board #{message.call(opts)}"
|
108
|
+
pipe_to(stream || :status_stream) { message.call(opts) }
|
109
|
+
sleep sleep_time
|
110
|
+
end
|
111
|
+
end
|
112
|
+
|
113
|
+
def status(id = @instance_id)
|
114
|
+
begin
|
115
|
+
ec2.describe_instances(dry_run: env('testing'), instance_ids: [id]).
|
116
|
+
reservations[0].instances[0].state.name
|
117
|
+
rescue Aws::EC2::Errors::DryRunOperation => e
|
118
|
+
logger.info "Dry run flag set for testing: #{format_error_message(e)}"
|
119
|
+
'testing'
|
120
|
+
end
|
121
|
+
end
|
122
|
+
|
123
|
+
def time_is_up?
|
124
|
+
# returns true when the hour mark approaches
|
125
|
+
an_hours_time = 60 * 60
|
126
|
+
five_minutes_time = 60 * 5
|
127
|
+
|
128
|
+
return false if run_time < five_minutes_time
|
129
|
+
run_time % an_hours_time < five_minutes_time
|
130
|
+
end
|
131
|
+
end
|
132
|
+
end
|
133
|
+
end
|
@@ -0,0 +1,33 @@
|
|
1
|
+
require 'singleton'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
class SmashError < Exception
|
6
|
+
include Singleton
|
7
|
+
|
8
|
+
attr_reader :errors, :error_collections
|
9
|
+
|
10
|
+
ErrorCollection = Struct.new(:type, :max, :storage) do
|
11
|
+
def push!(err_msg)
|
12
|
+
storage << err_msg
|
13
|
+
throw :die if storage[type].count >= max
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
def build!(*args)
|
18
|
+
|
19
|
+
@errors ||= args.inject({}) do |col, t|
|
20
|
+
col.merge(t => ErrCol.new(t, 5, []))
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def push_error!(type, message)
|
25
|
+
@errors[type].push!(message)
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
def initialize
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
@@ -0,0 +1,44 @@
|
|
1
|
+
require 'pathname'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module Storage
|
6
|
+
def source_task(file)
|
7
|
+
# TODO: better path management
|
8
|
+
# byebug TODO: replace this full path business
|
9
|
+
bucket = env('task storage')
|
10
|
+
unless task_path(file).exist?
|
11
|
+
objects = s3.list_objects(bucket: bucket).contents.select do |f|
|
12
|
+
/#{Regexp.escape file}/i =~ f.key
|
13
|
+
end
|
14
|
+
objects.each do |obj|
|
15
|
+
s3.get_object(bucket: bucket, key: obj.key, response_target: task_path(file))
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def search(bucket, pattern)
|
21
|
+
s3.list_objects(bucket: bucket).contents.select do |o|
|
22
|
+
o.key =~ pattern
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
def s3
|
27
|
+
@s3 ||= Aws::S3::Client.new(
|
28
|
+
region: region,
|
29
|
+
credentials: Auth.creds
|
30
|
+
)
|
31
|
+
end
|
32
|
+
|
33
|
+
def send_logs_to_s3
|
34
|
+
File.open(log_file) do |file|
|
35
|
+
s3.put_object(
|
36
|
+
bucket: log_bucket,
|
37
|
+
key: @instance_id,
|
38
|
+
body: file
|
39
|
+
)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,97 @@
|
|
1
|
+
require 'aws-sdk'
|
2
|
+
Aws.use_bundled_cert!
|
3
|
+
|
4
|
+
module Smash
|
5
|
+
module CloudPowers
|
6
|
+
module Synapse
|
7
|
+
module Pipe
|
8
|
+
def create_stream(name)
|
9
|
+
begin
|
10
|
+
config = stream_config(stream_name: env(name))
|
11
|
+
resp = kinesis.create_stream(config)
|
12
|
+
kinesis.wait_until(:stream_exists, stream_name: config[:stream_name])
|
13
|
+
resp.successful? # (http request successful && stream created)?
|
14
|
+
rescue Exception => e
|
15
|
+
if e.kind_of? Aws::Kinesis::Errors::ResourceInUseException
|
16
|
+
logger.info "#{name} already created"
|
17
|
+
stream_status = kinesis.describe_stream(name).stream_description.stream_status
|
18
|
+
return if stream_status == 'ACTIVE'
|
19
|
+
logger.info "Not ready for traffic. Wait for 30 seconds..."
|
20
|
+
sleep 30
|
21
|
+
nil # no request -> no response
|
22
|
+
else
|
23
|
+
error_message = format_error_message(e)
|
24
|
+
logger.error error_message
|
25
|
+
errors.push_error!(:ruby, error_message) # throws: :die, :failed_job
|
26
|
+
errors[:ruby] << error_message
|
27
|
+
false # the request was not successful
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
def flow_from_pipe(stream)
|
33
|
+
throw NotImplementedError
|
34
|
+
end
|
35
|
+
|
36
|
+
def flow_to_pipe(stream)
|
37
|
+
create_stream(stream) unless stream_exists? stream
|
38
|
+
records = yield if block_given?
|
39
|
+
body = message_body_collection(records)
|
40
|
+
# TODO: this isn't working yet. figure out retry logic
|
41
|
+
# resp = kinesis.put_records(body)
|
42
|
+
# retry(lambda { stream_exists? stream }) flow_to(stream)
|
43
|
+
@last_sequence_number = resp.records.map(&:sequence_number).sort.last
|
44
|
+
# TODO: what to return? true?
|
45
|
+
end
|
46
|
+
|
47
|
+
def from_pipe(stream)
|
48
|
+
# implemented get_records and/or other consuming app stuff
|
49
|
+
throw NotImplementedError
|
50
|
+
end
|
51
|
+
|
52
|
+
def kinesis
|
53
|
+
@kinesis ||= Aws::Kinesis::Client.new(
|
54
|
+
region: region,
|
55
|
+
credentials: Auth.creds,
|
56
|
+
)
|
57
|
+
end
|
58
|
+
|
59
|
+
def message_body_collection(records)
|
60
|
+
throw NotImplementedError
|
61
|
+
end
|
62
|
+
|
63
|
+
def pipe_message_body(opts = {})
|
64
|
+
{
|
65
|
+
stream_name: env(opts[:stream_name]) || env('status_stream'),
|
66
|
+
data: opts[:data] || update_message_body(opts),
|
67
|
+
partition_key: opts[:partition_key] || @instance_id
|
68
|
+
}
|
69
|
+
end
|
70
|
+
|
71
|
+
def pipe_to(stream)
|
72
|
+
create_stream(stream) unless stream_exists? stream
|
73
|
+
message = yield if block_given?
|
74
|
+
body = update_message_body(message)
|
75
|
+
resp = kinesis.put_record pipe_message_body(stream_name: stream, data: body.to_json)
|
76
|
+
# TODO: implement retry logic for failed request
|
77
|
+
@last_sequence_number = resp.sequence_number
|
78
|
+
end
|
79
|
+
|
80
|
+
def stream_config(opts = {})
|
81
|
+
config = {
|
82
|
+
stream_name: opts[:stream_name] || env('status_stream'),
|
83
|
+
shard_count: opts[:shard_count] || 1
|
84
|
+
}
|
85
|
+
end
|
86
|
+
|
87
|
+
def stream_exists?(name)
|
88
|
+
begin
|
89
|
+
kinesis.describe_stream(stream_name: env(name))
|
90
|
+
rescue Aws::Kinesis::Errors::ResourceNotFoundException => e
|
91
|
+
false
|
92
|
+
end
|
93
|
+
end
|
94
|
+
end
|
95
|
+
end
|
96
|
+
end
|
97
|
+
end
|
@@ -0,0 +1,112 @@
|
|
1
|
+
require 'uri'
|
2
|
+
require_relative '../helper'
|
3
|
+
|
4
|
+
module Smash
|
5
|
+
module CloudPowers
|
6
|
+
module Synapse
|
7
|
+
include Smash::CloudPowers::Helper
|
8
|
+
|
9
|
+
module Queue
|
10
|
+
Board = Struct.new(:set_name, :set_address, :workflow) do
|
11
|
+
include Smash::CloudPowers::Helper
|
12
|
+
def i_var
|
13
|
+
"@#{name}"
|
14
|
+
end
|
15
|
+
|
16
|
+
def address
|
17
|
+
set_address ||
|
18
|
+
env(name) ||
|
19
|
+
"https://sqs.us-west-2.amazonaws.com/088617881078/#{name}"
|
20
|
+
end
|
21
|
+
|
22
|
+
def name
|
23
|
+
set_name || address.split('/').last
|
24
|
+
end
|
25
|
+
|
26
|
+
def next_board
|
27
|
+
workflow.next
|
28
|
+
end
|
29
|
+
end # end Board
|
30
|
+
#############################################
|
31
|
+
|
32
|
+
def board_name(url)
|
33
|
+
url.to_s.split('/').last
|
34
|
+
end
|
35
|
+
|
36
|
+
# def board_name(url)
|
37
|
+
# # TODO: figure out a way to not have this and :name in Board
|
38
|
+
# # gets the name from the url
|
39
|
+
# if url =~ URI.regexp
|
40
|
+
# url = URI.parse(url)
|
41
|
+
# url.path.split('/').last.split('_').last
|
42
|
+
# else
|
43
|
+
# env(url)
|
44
|
+
# end
|
45
|
+
# end
|
46
|
+
|
47
|
+
def create_queue(name)
|
48
|
+
sqs.create_queue(queue_name: to_camel(name))
|
49
|
+
end
|
50
|
+
|
51
|
+
def delete_queue_message(queue, opts = {})
|
52
|
+
poll(queue, opts) do |msg, stats|
|
53
|
+
poller(queue).delete_message(msg)
|
54
|
+
throw :stop_polling
|
55
|
+
end
|
56
|
+
end
|
57
|
+
|
58
|
+
def get_count(board)
|
59
|
+
sqs.get_queue_attributes(
|
60
|
+
queue_url: board_name(board),
|
61
|
+
attribute_names: ['ApproximateNumberOfMessages']
|
62
|
+
).attributes['ApproximateNumberOfMessages'].to_f
|
63
|
+
end
|
64
|
+
|
65
|
+
# Params: board<string>
|
66
|
+
# returns a message and deletes it from its origin
|
67
|
+
def pluck_message(board)
|
68
|
+
poll(board) do |msg, poller|
|
69
|
+
poller.delete_message(msg)
|
70
|
+
return msg
|
71
|
+
end
|
72
|
+
end
|
73
|
+
|
74
|
+
def poll(board, opts = {})
|
75
|
+
this_poller = poller(board)
|
76
|
+
this_poller.poll(opts) do |msg|
|
77
|
+
yield msg, this_poller if block_given?
|
78
|
+
end
|
79
|
+
end
|
80
|
+
|
81
|
+
def poller(board_name)
|
82
|
+
board = Board.new(board_name)
|
83
|
+
|
84
|
+
unless instance_variable_defined?(board.i_var)
|
85
|
+
instance_variable_set(
|
86
|
+
board.i_var,
|
87
|
+
Aws::SQS::QueuePoller.new(board.address)
|
88
|
+
)
|
89
|
+
end
|
90
|
+
instance_variable_get(board.i_var)
|
91
|
+
end
|
92
|
+
|
93
|
+
def queue_exists?(name)
|
94
|
+
sqs.list_queues(queue_name_prefix: name)
|
95
|
+
end
|
96
|
+
|
97
|
+
def send_queue_message(message, *board_info)
|
98
|
+
board = board_info.first.kind_of?(Board) ? board_info.first : Board.new(*board_info)
|
99
|
+
message = message.to_json unless message.kind_of? String
|
100
|
+
sqs.send_message(
|
101
|
+
queue_url: board.address,
|
102
|
+
message_body: message
|
103
|
+
)
|
104
|
+
end
|
105
|
+
|
106
|
+
def sqs
|
107
|
+
@sqs ||= Aws::SQS::Client.new(credentials: Auth.creds)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
module Smash
|
2
|
+
class Workflow
|
3
|
+
attr_accessor :all_states, :first, :last, :previous, :states
|
4
|
+
|
5
|
+
def initialize(statez = nil)
|
6
|
+
# TODO: figure out wtf here with the i-vars changing each other
|
7
|
+
@all_states = statez || [:backlog, :wip, :done]
|
8
|
+
@states = statez || [:backlog, :wip, :done]
|
9
|
+
@first = (statez || [:backlog, :wip, :done]).first
|
10
|
+
@last = (statez || [:backlog, :wip, :done]).last
|
11
|
+
@previous = [(statez || [:backlog, :wip, :done]).first]
|
12
|
+
end
|
13
|
+
|
14
|
+
def current
|
15
|
+
@states.first
|
16
|
+
end
|
17
|
+
|
18
|
+
def finished?
|
19
|
+
@states.first == @states.last
|
20
|
+
end
|
21
|
+
|
22
|
+
def next
|
23
|
+
finished ? @states.first : @states[1]
|
24
|
+
end
|
25
|
+
|
26
|
+
def next!
|
27
|
+
return if finished?
|
28
|
+
@previous << @states.shift
|
29
|
+
@states.first
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
metadata
ADDED
@@ -0,0 +1,112 @@
|
|
1
|
+
--- !ruby/object:Gem::Specification
|
2
|
+
name: cloud_powers
|
3
|
+
version: !ruby/object:Gem::Version
|
4
|
+
version: 0.1.1
|
5
|
+
platform: ruby
|
6
|
+
authors:
|
7
|
+
- Adam Phillipps
|
8
|
+
autorequire:
|
9
|
+
bindir: exe
|
10
|
+
cert_chain: []
|
11
|
+
date: 2016-09-23 00:00:00.000000000 Z
|
12
|
+
dependencies:
|
13
|
+
- !ruby/object:Gem::Dependency
|
14
|
+
name: bundler
|
15
|
+
requirement: !ruby/object:Gem::Requirement
|
16
|
+
requirements:
|
17
|
+
- - "~>"
|
18
|
+
- !ruby/object:Gem::Version
|
19
|
+
version: '1.12'
|
20
|
+
type: :development
|
21
|
+
prerelease: false
|
22
|
+
version_requirements: !ruby/object:Gem::Requirement
|
23
|
+
requirements:
|
24
|
+
- - "~>"
|
25
|
+
- !ruby/object:Gem::Version
|
26
|
+
version: '1.12'
|
27
|
+
- !ruby/object:Gem::Dependency
|
28
|
+
name: rake
|
29
|
+
requirement: !ruby/object:Gem::Requirement
|
30
|
+
requirements:
|
31
|
+
- - "~>"
|
32
|
+
- !ruby/object:Gem::Version
|
33
|
+
version: '10.0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - "~>"
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '10.0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rspec
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - "~>"
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '3.0'
|
48
|
+
type: :development
|
49
|
+
prerelease: false
|
50
|
+
version_requirements: !ruby/object:Gem::Requirement
|
51
|
+
requirements:
|
52
|
+
- - "~>"
|
53
|
+
- !ruby/object:Gem::Version
|
54
|
+
version: '3.0'
|
55
|
+
description: CloudPowers is a wrapper around AWS and other cloud services. This wrapper
|
56
|
+
was developed specifically for the Brain project but can be used in any other ruby
|
57
|
+
project.
|
58
|
+
email:
|
59
|
+
- adam.phillipps@gmail.com
|
60
|
+
executables: []
|
61
|
+
extensions: []
|
62
|
+
extra_rdoc_files: []
|
63
|
+
files:
|
64
|
+
- ".gitignore"
|
65
|
+
- ".rspec"
|
66
|
+
- ".travis.yml"
|
67
|
+
- Gemfile
|
68
|
+
- Gemfile.lock
|
69
|
+
- LICENSE.txt
|
70
|
+
- README.md
|
71
|
+
- Rakefile
|
72
|
+
- bin/console
|
73
|
+
- bin/setup
|
74
|
+
- cloud_powers.gemspec
|
75
|
+
- lib/cloud_powers.rb
|
76
|
+
- lib/cloud_powers/auth.rb
|
77
|
+
- lib/cloud_powers/aws_resources.rb
|
78
|
+
- lib/cloud_powers/delegator.rb
|
79
|
+
- lib/cloud_powers/helper.rb
|
80
|
+
- lib/cloud_powers/self_awareness.rb
|
81
|
+
- lib/cloud_powers/smash_error.rb
|
82
|
+
- lib/cloud_powers/storage.rb
|
83
|
+
- lib/cloud_powers/synapse/pipe.rb
|
84
|
+
- lib/cloud_powers/synapse/queue.rb
|
85
|
+
- lib/cloud_powers/synapse/synapse.rb
|
86
|
+
- lib/cloud_powers/version.rb
|
87
|
+
- lib/cloud_powers/workflow.rb
|
88
|
+
homepage: https://github.com/adam-phillipps/cloud_powers
|
89
|
+
licenses:
|
90
|
+
- MIT
|
91
|
+
metadata: {}
|
92
|
+
post_install_message:
|
93
|
+
rdoc_options: []
|
94
|
+
require_paths:
|
95
|
+
- lib
|
96
|
+
required_ruby_version: !ruby/object:Gem::Requirement
|
97
|
+
requirements:
|
98
|
+
- - ">="
|
99
|
+
- !ruby/object:Gem::Version
|
100
|
+
version: '0'
|
101
|
+
required_rubygems_version: !ruby/object:Gem::Requirement
|
102
|
+
requirements:
|
103
|
+
- - ">="
|
104
|
+
- !ruby/object:Gem::Version
|
105
|
+
version: '0'
|
106
|
+
requirements: []
|
107
|
+
rubyforge_project:
|
108
|
+
rubygems_version: 2.5.1
|
109
|
+
signing_key:
|
110
|
+
specification_version: 4
|
111
|
+
summary: Cloud providers wrapper. Currently only AWS is supported.
|
112
|
+
test_files: []
|