cloud_powers 1.1.0 → 1.1.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/cloud_powers.gemspec +1 -1
- data/lib/cloud_powers.rb +4 -1
- data/lib/cloud_powers/helpers/logic_help.rb +26 -0
- data/lib/cloud_powers/helpers/path_help.rb +1 -1
- data/lib/cloud_powers/resource.rb +1 -0
- data/lib/cloud_powers/storage.rb +25 -25
- data/lib/cloud_powers/storage/bucket.rb +118 -46
- data/lib/cloud_powers/storage/local.rb +15 -2
- data/lib/cloud_powers/{synapse/synapse.rb → synapse.rb} +0 -0
- data/lib/cloud_powers/synapse/broadcast.rb +4 -1
- data/lib/cloud_powers/synapse/pipe.rb +23 -7
- data/lib/cloud_powers/synapse/queue.rb +18 -4
- data/lib/cloud_powers/synapse/queue/board.rb +1 -0
- data/lib/cloud_powers/version.rb +1 -1
- metadata +7 -9
- data/lib/cloud_powers/example_objects.rb +0 -88
- data/lib/cloud_powers/self_awareness.rb +0 -240
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79e380ab917b71621e00f9877499d22ec6cda41a
|
4
|
+
data.tar.gz: ffe5f9828c11ddce25c1c5c7fcb91d70d4257dfa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ec413e2c2062102cd740c34c85524070bedb3b8fb19223065ae9927d2975293a2bb655f296677e03f8508f2645dce44387565c0d1c47a33167c3e9c4d12d82c
|
7
|
+
data.tar.gz: 36c347b259c84897e6cba930769cba4670cd41fcf4c32423608e922e27be67079ad6956b9355dce19fe31c0c36981716f0ef48a516b604ccebc7d4672b012f11
|
data/cloud_powers.gemspec
CHANGED
@@ -43,7 +43,7 @@ Gem::Specification.new do |spec|
|
|
43
43
|
spec.add_runtime_dependency 'zip-zip', '~> 0.3'
|
44
44
|
spec.add_runtime_dependency 'websocket-eventmachine-server', '~> 1.0.1'
|
45
45
|
spec.add_runtime_dependency 'websocket-eventmachine-client', '~> 1.2.0'
|
46
|
-
spec.add_runtime_dependency 'workflow'
|
46
|
+
spec.add_runtime_dependency 'workflow', '~> 1.2'
|
47
47
|
|
48
48
|
spec.add_development_dependency 'bundler', '~> 1.14.3'
|
49
49
|
spec.add_development_dependency 'byebug', '~> 9'
|
data/lib/cloud_powers.rb
CHANGED
@@ -4,8 +4,9 @@ require 'cloud_powers/helpers'
|
|
4
4
|
require 'cloud_powers/node'
|
5
5
|
require 'cloud_powers/resource'
|
6
6
|
require 'cloud_powers/storage'
|
7
|
+
require 'cloud_powers/synapse'
|
7
8
|
require 'cloud_powers/version'
|
8
|
-
require 'cloud_powers/
|
9
|
+
require 'cloud_powers/zenv'
|
9
10
|
|
10
11
|
# The Smash module allows us to use CloudPowers under a shared name space with other projects.
|
11
12
|
module Smash
|
@@ -21,5 +22,7 @@ module Smash
|
|
21
22
|
include Smash::CloudPowers::Synapse
|
22
23
|
# CRUD on Nodes, which are individual instances
|
23
24
|
include Smash::CloudPowers::Node
|
25
|
+
# Project information and tie-ins through dynamic lookups and caching
|
26
|
+
include Smash::CloudPowers::Zenv
|
24
27
|
end
|
25
28
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
1
3
|
module Smash
|
2
4
|
module CloudPowers
|
3
5
|
module LogicHelp
|
@@ -166,6 +168,30 @@ module Smash
|
|
166
168
|
extraInfo: updated_extra_info
|
167
169
|
}.merge(opts)
|
168
170
|
end
|
171
|
+
|
172
|
+
# Wait until a block finishes or a given time ellapses.
|
173
|
+
#
|
174
|
+
# Parameters
|
175
|
+
# * +wait_time+ +Integer+ - time to wait before returning
|
176
|
+
# * <tt>&block</tt> - block to execute. a <i>"truthy"</tt> result
|
177
|
+
# will finish this method
|
178
|
+
#
|
179
|
+
# Returns
|
180
|
+
# * +nil+ - if the time ellapses or your block returns nil
|
181
|
+
# * +Object+ - if your block succeeds, its return value is returned
|
182
|
+
#
|
183
|
+
# Example
|
184
|
+
# wait_until(3) { sleep 1; 'bla' }
|
185
|
+
# => 'bla' # returns after 1 second, not 3
|
186
|
+
#
|
187
|
+
# wait_until(1) { sleep 3; 'bla' }
|
188
|
+
# => nil # returns after 1 second because the block never finished
|
189
|
+
def wait_until(wait_time = 5)
|
190
|
+
begin
|
191
|
+
Timeout::timeout(wait_time) { block_given? ? (yield) : (sleep wait_time) }
|
192
|
+
rescue Timeout::Error => e
|
193
|
+
end
|
194
|
+
end
|
169
195
|
end
|
170
196
|
end
|
171
197
|
end
|
@@ -140,7 +140,7 @@ module Smash
|
|
140
140
|
|
141
141
|
def paths_gcd(*paths)
|
142
142
|
start_from = paths_lcd(*paths)
|
143
|
-
paths.sort.last.relative_path_from(start_from)
|
143
|
+
start_from + paths.sort.last.relative_path_from(start_from)
|
144
144
|
end
|
145
145
|
|
146
146
|
# Find all <i>directories</i> that match the given string.
|
data/lib/cloud_powers/storage.rb
CHANGED
@@ -113,21 +113,11 @@ module Smash
|
|
113
113
|
# | |_custom_greetings.js # could be an after effects JS script
|
114
114
|
def source_job(file)
|
115
115
|
storage_delegates(:bucket).map do |delegate|
|
116
|
-
|
117
|
-
|
118
|
-
|
116
|
+
# TODO!! Death to hardcoding
|
117
|
+
# Let's clean this up, eh?
|
118
|
+
loc = zfind('jobs') || 'jobs'
|
119
|
+
delegate.find(file, location: loc) { |f, data| f.write data }
|
119
120
|
end.flatten
|
120
|
-
# TODO: better path management
|
121
|
-
# bucket = zfind('jobs storage')
|
122
|
-
# if job_path(to_ruby_file_name(file)).nil?
|
123
|
-
# objects = s3.list_objects(bucket: bucket).contents.select do |f|
|
124
|
-
# /#{Regexp.escape file}/i =~ f.key
|
125
|
-
# end
|
126
|
-
|
127
|
-
# objects.each do |obj|
|
128
|
-
# s3.get_object(bucket: bucket, key: obj.key, response_target: job_path(file))
|
129
|
-
# end
|
130
|
-
# end
|
131
121
|
end
|
132
122
|
|
133
123
|
# Search through a bucket to find a file, based on a regex
|
@@ -165,6 +155,10 @@ module Smash
|
|
165
155
|
#
|
166
156
|
# Parameters
|
167
157
|
# * name +String+ - name of the storage you want to interact with
|
158
|
+
# * type +Symbol+ - the type of storage you want to build. The default
|
159
|
+
# is +:local+
|
160
|
+
# * +config+ +KeywordArgument+s - other configuration can be passed to
|
161
|
+
# customize the object just after initialization
|
168
162
|
#
|
169
163
|
# Returns
|
170
164
|
# Storage::[Local|Bucket]
|
@@ -176,8 +170,12 @@ module Smash
|
|
176
170
|
def build_storage(name:, type: nil, **config)
|
177
171
|
type = to_pascal(type || :local)
|
178
172
|
|
179
|
-
storage_resource = Smash::CloudPowers::Storage.const_get(type).
|
180
|
-
|
173
|
+
storage_resource = Smash::CloudPowers::Storage.const_get(type).build(
|
174
|
+
name: to_camel(name),
|
175
|
+
type: (type || :storage),
|
176
|
+
project_root: config[:project_root] || project_root,
|
177
|
+
**config
|
178
|
+
)
|
181
179
|
|
182
180
|
yield storage_resource if block_given?
|
183
181
|
|
@@ -206,12 +204,14 @@ module Smash
|
|
206
204
|
# get_queue_message_count
|
207
205
|
def create_storage(name:, type: nil, **config)
|
208
206
|
# default is :local
|
209
|
-
|
207
|
+
storage_resource = Smash::CloudPowers::Storage.const_get(type).create(
|
208
|
+
name: to_camel(name),
|
209
|
+
type: (type || :storage),
|
210
|
+
project_root: config[:project_root] || project_root,
|
211
|
+
**config
|
212
|
+
)
|
210
213
|
|
211
|
-
storage_resource
|
212
|
-
Smash::CloudPowers::Storage.const_get(to_pascal(type)).create!(
|
213
|
-
name: to_camel(name), **config
|
214
|
-
)
|
214
|
+
yield storage_resource if block_given?
|
215
215
|
|
216
216
|
attr_map(storage_resource.call_name => storage_resource) do |attribute, resource|
|
217
217
|
instance_attr_accessor attribute
|
@@ -334,11 +334,11 @@ module Smash
|
|
334
334
|
if respond_to? delegate_call_name
|
335
335
|
public_send delegate_call_name
|
336
336
|
else
|
337
|
-
config = storage_config(type).merge(
|
337
|
+
config = storage_config(type).merge(
|
338
|
+
{ name: 'delegate', type: type, project_root: project_root }
|
339
|
+
)
|
338
340
|
# set delegates up to search everywhere
|
339
|
-
build_storage(**config)
|
340
|
-
storage.tie_in_path = storage.origin
|
341
|
-
end
|
341
|
+
build_storage(**config)
|
342
342
|
end
|
343
343
|
end
|
344
344
|
end
|
@@ -7,16 +7,21 @@ module Smash
|
|
7
7
|
class Bucket < Smash::CloudPowers::Resource
|
8
8
|
include Smash::CloudPowers::Storage
|
9
9
|
|
10
|
-
attr_accessor :origin, :delimiter, :tie_in_path
|
10
|
+
attr_accessor :origin, :delimiter, :tie_in_path, :s3
|
11
11
|
|
12
12
|
def initialize(name:, client: s3, **config)
|
13
13
|
super
|
14
|
+
@s3 = client
|
14
15
|
@bucket = Aws::S3::Bucket.new(name: name, client: client)
|
15
|
-
|
16
|
+
@origin = config[:origin] || project_root.split.last
|
17
|
+
build_storage(
|
18
|
+
name: 'local',
|
19
|
+
named_type: 'storage',
|
20
|
+
project_root: @project_root
|
21
|
+
).link
|
16
22
|
@delimiter = config[:delimiter] || common_delimiter
|
17
23
|
# should be the 'lowest level' of storage this object is aware of
|
18
|
-
@
|
19
|
-
@tie_in_path = paths_gcd(local_storage.tie_in_path, project_root)
|
24
|
+
@tie_in_path = local_storage.tie_in_path
|
20
25
|
end
|
21
26
|
|
22
27
|
def self.tie_in_config
|
@@ -33,7 +38,9 @@ module Smash
|
|
33
38
|
# end
|
34
39
|
|
35
40
|
def create_resource
|
36
|
-
|
41
|
+
# TODO! Use Seahorse #on_complete, #on_success etc. callbacks
|
42
|
+
@response = s3.create_bucket(bucket: name).data
|
43
|
+
@remote_origin = @response.location.gsub(%r"^\/",'')
|
37
44
|
yield self if block_given?
|
38
45
|
self
|
39
46
|
end
|
@@ -43,6 +50,53 @@ module Smash
|
|
43
50
|
!!(@bucket.exists? rescue nil)
|
44
51
|
end
|
45
52
|
|
53
|
+
# Find the closest match to the request you're looking for with those
|
54
|
+
# params. If you pass this method a block, a file will be created and
|
55
|
+
# passed to the block, along with the data from the file that was found
|
56
|
+
# in the bucket.
|
57
|
+
#
|
58
|
+
# Parameters
|
59
|
+
# * +patterns+ +String+|+Regexp+ - names of files or buckets to look for
|
60
|
+
# * +location+ +String+ - narrow down the scope of the search
|
61
|
+
# * +opts+ +KeywordArgument+(s) - pass more specific configuration along
|
62
|
+
# to the search method(s) and object(s) used
|
63
|
+
#
|
64
|
+
# Returns
|
65
|
+
# * +String+:
|
66
|
+
# * if no +block+ was passed, only the name of the object found is
|
67
|
+
# returned
|
68
|
+
# * if a +block+ is passed to this method, you will have a +file+ and
|
69
|
+
# +data+ object to use as params. The file is created at the
|
70
|
+
# location of this +Storage+ object's +origin+, which is defaulted
|
71
|
+
# to +project_root+/zlib/+@name+_bucket. The +data+ object is the
|
72
|
+
# data from the object in the bucket
|
73
|
+
def find(*patterns, location: origin, **opts)
|
74
|
+
path = select(*patterns, location: location, **opts).first
|
75
|
+
file_name = path.kind_of?(Pathname) ? path.split.last.to_s : path
|
76
|
+
file_permissions = opts[:file_permissions] || 'a+'
|
77
|
+
path = local_storage.tie_in_path + path
|
78
|
+
|
79
|
+
if block_given?
|
80
|
+
begin
|
81
|
+
to_realpath(path)
|
82
|
+
File.open(path, file_permissions) do |file|
|
83
|
+
data = get_file(file_name, location: location).get.body.read
|
84
|
+
yield file, data
|
85
|
+
end
|
86
|
+
rescue Exception => e
|
87
|
+
byebug
|
88
|
+
if e =~ /no implicit conversion of nil into String/
|
89
|
+
logger.info "no file found from #{path} called #{file}"
|
90
|
+
nil
|
91
|
+
else
|
92
|
+
raise
|
93
|
+
end
|
94
|
+
end
|
95
|
+
else
|
96
|
+
path
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
46
100
|
# Find out if a job exists in the bucket
|
47
101
|
#
|
48
102
|
# Parameters
|
@@ -59,6 +113,7 @@ module Smash
|
|
59
113
|
|
60
114
|
def link
|
61
115
|
if exists?
|
116
|
+
byebug
|
62
117
|
# do stuff
|
63
118
|
else
|
64
119
|
save!
|
@@ -67,60 +122,74 @@ module Smash
|
|
67
122
|
@linked = @response.nil?
|
68
123
|
end
|
69
124
|
|
70
|
-
def select(*args, **opts)
|
71
|
-
|
72
|
-
|
125
|
+
# def select(*args, **opts)
|
126
|
+
# pattern = args.pop
|
127
|
+
# location = args.shift || origin
|
73
128
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
129
|
+
# names = if filename?(pattern)
|
130
|
+
# get_file_names(pattern, location: location)
|
131
|
+
# else
|
132
|
+
# get_bucket_names(pattern, location: location)
|
133
|
+
# end
|
79
134
|
|
80
135
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
end
|
136
|
+
# names = get_file_names(pattern, location: location)
|
137
|
+
# if block_given?
|
138
|
+
# names.map do |file_name|
|
139
|
+
# new_file = to_realpath(local_storage.find(file_name))
|
140
|
+
# File.open(new_file, 'r+') do |file|
|
141
|
+
# data = get_file(file_name, location: location).get.body.read
|
142
|
+
# yield file, data
|
143
|
+
# end
|
144
|
+
# end
|
145
|
+
# else
|
146
|
+
# names
|
147
|
+
# end
|
148
|
+
# end
|
149
|
+
def select(*patterns, location: @tie_in_path, **opts)
|
150
|
+
file_rights = opts[:file_rights] || 'a+'
|
151
|
+
names = patterns.map do |pattern|
|
152
|
+
pattern = pattern.kind_of?(Regexp) ? pattern : %r"#{pattern}"
|
94
153
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
154
|
+
# if filename?(pattern.source)
|
155
|
+
# get_file_names(pattern, location: location)
|
156
|
+
# else
|
157
|
+
# get_bucket_names(pattern, location: location)
|
158
|
+
# end
|
159
|
+
files_thread = Thread.new do
|
160
|
+
get_file_names(pattern, location: location)
|
161
|
+
end
|
162
|
+
bucket_thread = Thread.new do
|
163
|
+
get_bucket_names(pattern, location: location)
|
164
|
+
end
|
165
|
+
files_thread.join
|
166
|
+
bucket_thread.join
|
100
167
|
|
168
|
+
[files_thread.value, bucket_thread.value]
|
169
|
+
end.flatten
|
101
170
|
if block_given?
|
102
|
-
|
103
|
-
|
171
|
+
names.map do |file_name|
|
172
|
+
new_file = local_storage.touch(file_name)
|
173
|
+
File.open(new_file, file_rights) do |file|
|
104
174
|
data = get_file(file_name, location: location).get.body.read
|
105
175
|
yield file, data
|
106
176
|
end
|
107
|
-
rescue Exception => e
|
108
|
-
if e =~ /no implicit conversion of nil into String/
|
109
|
-
logger.info "no file found from #{path} called #{file}"
|
110
|
-
nil
|
111
|
-
else
|
112
|
-
super
|
113
|
-
end
|
114
177
|
end
|
115
178
|
else
|
116
|
-
|
179
|
+
names
|
117
180
|
end
|
118
181
|
end
|
119
182
|
|
120
183
|
# Notes:
|
121
|
-
# * See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3
|
122
|
-
|
123
|
-
|
184
|
+
# * See <i>https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/</i>
|
185
|
+
# <i>Client.html#put_object-instance_method</i>
|
186
|
+
def put(file_name, location = local_storage.tie_in_path, **config)
|
187
|
+
body = local_storage.find(
|
188
|
+
file_name,
|
189
|
+
location: location,
|
190
|
+
file_rights: 'r'
|
191
|
+
) { |data| data }
|
192
|
+
|
124
193
|
return if body.nil?
|
125
194
|
|
126
195
|
client.put_object(
|
@@ -129,6 +198,9 @@ module Smash
|
|
129
198
|
true
|
130
199
|
end
|
131
200
|
|
201
|
+
########################################################################
|
202
|
+
# PRIVATE METHODS
|
203
|
+
########################################################################
|
132
204
|
private
|
133
205
|
# Get a local file using the file name and optional location
|
134
206
|
#
|
@@ -138,7 +210,7 @@ module Smash
|
|
138
210
|
#
|
139
211
|
# Returns
|
140
212
|
# +String+|+IO+
|
141
|
-
def get_local_file_body(file, location =
|
213
|
+
def get_local_file_body(file, location = local_storage.tie_in_path)
|
142
214
|
scope = paths_gcd(tie_in_path, location)
|
143
215
|
local_storage.
|
144
216
|
find(file, location: scope, file_rights: 'r') { |data| data }
|
@@ -169,7 +241,7 @@ module Smash
|
|
169
241
|
end
|
170
242
|
|
171
243
|
def get_bucket_names(pattern, location: origin)
|
172
|
-
buckets = client.list_buckets.buckets.map(&:name)
|
244
|
+
buckets = client.list_buckets.buckets.map(&:name).grep(pattern)
|
173
245
|
end
|
174
246
|
|
175
247
|
# Get the actual object using S3::Bucket
|
@@ -11,7 +11,8 @@ module Smash
|
|
11
11
|
|
12
12
|
def initialize(name:, origin: zlib_home, **config)
|
13
13
|
super
|
14
|
-
|
14
|
+
tmp_origin = origin.kind_of?(Pathname) ? origin : to_pathname(origin)
|
15
|
+
@origin = paths_gcd(tmp_origin, project_root)
|
15
16
|
@tie_in_path = to_pathname(@origin, to_camel(name))
|
16
17
|
end
|
17
18
|
|
@@ -96,7 +97,6 @@ module Smash
|
|
96
97
|
grep(regex).
|
97
98
|
map { |path| to_realpath(path) }
|
98
99
|
end.flatten
|
99
|
-
|
100
100
|
if block_given?
|
101
101
|
paths.map { |path| yield File.open(path, file_rights) }
|
102
102
|
else
|
@@ -123,6 +123,19 @@ module Smash
|
|
123
123
|
path
|
124
124
|
end
|
125
125
|
end
|
126
|
+
|
127
|
+
def touch(path)
|
128
|
+
if @tie_in_path.nil?
|
129
|
+
logger.debug("Local#touch(#{path}) called")
|
130
|
+
super(path)
|
131
|
+
else
|
132
|
+
scope, last_piece = to_pathname(path).split
|
133
|
+
tied_in_path = paths_gcd(@tie_in_path, scope.expand_path) +
|
134
|
+
(filename?(last_piece) ? last_piece : '')
|
135
|
+
logger.debug("Local#touch(#{tied_in_path}) called")
|
136
|
+
super(tied_in_path)
|
137
|
+
end
|
138
|
+
end
|
126
139
|
end
|
127
140
|
end
|
128
141
|
end
|
File without changes
|
@@ -48,7 +48,10 @@ module Smash
|
|
48
48
|
def create_channel(name, **config)
|
49
49
|
channel_resource =
|
50
50
|
Smash::CloudPowers::Synapse::Broadcast::Channel.create!(
|
51
|
-
name: name,
|
51
|
+
name: name,
|
52
|
+
client: sns,
|
53
|
+
project_root: project_root,
|
54
|
+
**config
|
52
55
|
)
|
53
56
|
|
54
57
|
self.attr_map(channel_resource.call_name => channel_resource) do |attribute, resource|
|
@@ -8,21 +8,29 @@ module Smash
|
|
8
8
|
include Smash::CloudPowers::Helpers
|
9
9
|
include Smash::CloudPowers::Zenv
|
10
10
|
|
11
|
-
def build_pipe(name:, type: :stream, **config)
|
11
|
+
def build_pipe(name:, type: :stream, project_root: project_root, **config)
|
12
12
|
build_method_name = "build_#{type}"
|
13
13
|
if self.respond_to? build_method_name
|
14
|
-
self.public_send
|
14
|
+
self.public_send(build_method_name,
|
15
|
+
name: name,
|
16
|
+
project_root: project_root,
|
17
|
+
**config
|
18
|
+
)
|
15
19
|
else
|
16
|
-
build_stream(name: name, **config)
|
20
|
+
build_stream(name: name, project_root: project_root, **config)
|
17
21
|
end
|
18
22
|
end
|
19
23
|
|
20
24
|
def create_pipe(name:, type: :stream, **config)
|
21
25
|
create_method_name = "create_#{type}"
|
22
26
|
if self.respond_to? create_method_name
|
23
|
-
self.public_send
|
27
|
+
self.public_send(create_method_name,
|
28
|
+
name: name,
|
29
|
+
project_root: config[:project_root] || project_root,
|
30
|
+
**config
|
31
|
+
)
|
24
32
|
else
|
25
|
-
create_stream(name: name, **config)
|
33
|
+
create_stream(name: name, project_root: project_root, **config)
|
26
34
|
end
|
27
35
|
end
|
28
36
|
|
@@ -38,7 +46,10 @@ module Smash
|
|
38
46
|
# * returns false if the stream was not created
|
39
47
|
def build_stream(name:, client: kinesis, **config)
|
40
48
|
stream_resource = Smash::CloudPowers::Synapse::Pipe::Stream.build(
|
41
|
-
name: name,
|
49
|
+
name: name,
|
50
|
+
client: client,
|
51
|
+
project_root: config[:project_root] || project_root,
|
52
|
+
**config
|
42
53
|
)
|
43
54
|
|
44
55
|
self.attr_map(stream_resource.call_name => stream_resource) do |attribute, resource|
|
@@ -51,7 +62,12 @@ module Smash
|
|
51
62
|
|
52
63
|
def create_stream(name:, client: kinesis, **config)
|
53
64
|
stream_resource =
|
54
|
-
Smash::CloudPowers::Synapse::Pipe::Stream.create!(
|
65
|
+
Smash::CloudPowers::Synapse::Pipe::Stream.create!(
|
66
|
+
name: name,
|
67
|
+
client: client,
|
68
|
+
|
69
|
+
**config
|
70
|
+
)
|
55
71
|
|
56
72
|
self.attr_map(stream_resource.call_name => stream_resource) do |attribute, resource|
|
57
73
|
instance_attr_accessor attribute
|
@@ -112,7 +112,11 @@ module Smash
|
|
112
112
|
def build_queue(name:, type: :board, **config)
|
113
113
|
build_method_name = "build_#{type}"
|
114
114
|
if self.respond_to? build_method_name
|
115
|
-
self.public_send
|
115
|
+
self.public_send(build_method_name,
|
116
|
+
name: name,
|
117
|
+
project_root: config[:project_root] || project_root,
|
118
|
+
**config
|
119
|
+
)
|
116
120
|
else
|
117
121
|
build_board(name: name, **config)
|
118
122
|
end
|
@@ -149,7 +153,11 @@ module Smash
|
|
149
153
|
def create_queue(name:, type: :board, **config)
|
150
154
|
create_method_name = "build_#{type}"
|
151
155
|
if self.respond_to? create_method_name
|
152
|
-
self.public_send
|
156
|
+
self.public_send(create_method_name,
|
157
|
+
name: name,
|
158
|
+
project_root: config[:project_root] || project_root,
|
159
|
+
**config
|
160
|
+
)
|
153
161
|
else
|
154
162
|
create_queue(name: name, **config)
|
155
163
|
end
|
@@ -171,7 +179,10 @@ module Smash
|
|
171
179
|
# => https://sqs.us-west-2.amazonaws.com/81234567/exampleQueue
|
172
180
|
def build_board(name:, client: sqs, **config)
|
173
181
|
board_resource = Smash::CloudPowers::Synapse::Queue::Board.build(
|
174
|
-
name: to_camel(name),
|
182
|
+
name: to_camel(name),
|
183
|
+
client: client,
|
184
|
+
project_root: config[:project_root] || project_root,
|
185
|
+
**config
|
175
186
|
)
|
176
187
|
|
177
188
|
attr_map(board_resource.call_name => board_resource) do |attribute, resource|
|
@@ -195,7 +206,10 @@ module Smash
|
|
195
206
|
# get_queue_message_count
|
196
207
|
def create_board(name:, client: sqs, **config)
|
197
208
|
board_resource = Smash::CloudPowers::Synapse::Queue::Board.create!(
|
198
|
-
name: to_camel(name),
|
209
|
+
name: to_camel(name),
|
210
|
+
client: sqs,
|
211
|
+
project_root: config[:project_root] || project_root,
|
212
|
+
**config
|
199
213
|
)
|
200
214
|
|
201
215
|
attr_map(board_resource.call_name => board_resource) do |attribute, resource|
|
data/lib/cloud_powers/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloud_powers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Phillipps
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-03-
|
13
|
+
date: 2017-03-27 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport-core-ext
|
@@ -128,16 +128,16 @@ dependencies:
|
|
128
128
|
name: workflow
|
129
129
|
requirement: !ruby/object:Gem::Requirement
|
130
130
|
requirements:
|
131
|
-
- - "
|
131
|
+
- - "~>"
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: '
|
133
|
+
version: '1.2'
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
137
137
|
requirements:
|
138
|
-
- - "
|
138
|
+
- - "~>"
|
139
139
|
- !ruby/object:Gem::Version
|
140
|
-
version: '
|
140
|
+
version: '1.2'
|
141
141
|
- !ruby/object:Gem::Dependency
|
142
142
|
name: bundler
|
143
143
|
requirement: !ruby/object:Gem::Requirement
|
@@ -236,7 +236,6 @@ files:
|
|
236
236
|
- lib/cloud_powers/auth.rb
|
237
237
|
- lib/cloud_powers/aws_resources.rb
|
238
238
|
- lib/cloud_powers/creatable.rb
|
239
|
-
- lib/cloud_powers/example_objects.rb
|
240
239
|
- lib/cloud_powers/helpers.rb
|
241
240
|
- lib/cloud_powers/helpers/lang_help.rb
|
242
241
|
- lib/cloud_powers/helpers/logic_help.rb
|
@@ -244,12 +243,12 @@ files:
|
|
244
243
|
- lib/cloud_powers/node.rb
|
245
244
|
- lib/cloud_powers/node/instance.rb
|
246
245
|
- lib/cloud_powers/resource.rb
|
247
|
-
- lib/cloud_powers/self_awareness.rb
|
248
246
|
- lib/cloud_powers/smash_error.rb
|
249
247
|
- lib/cloud_powers/storage.rb
|
250
248
|
- lib/cloud_powers/storage/bucket.rb
|
251
249
|
- lib/cloud_powers/storage/local.rb
|
252
250
|
- lib/cloud_powers/stubs/aws_stubs.rb
|
251
|
+
- lib/cloud_powers/synapse.rb
|
253
252
|
- lib/cloud_powers/synapse/broadcast.rb
|
254
253
|
- lib/cloud_powers/synapse/broadcast/channel.rb
|
255
254
|
- lib/cloud_powers/synapse/pipe.rb
|
@@ -258,7 +257,6 @@ files:
|
|
258
257
|
- lib/cloud_powers/synapse/queue.rb
|
259
258
|
- lib/cloud_powers/synapse/queue/board.rb
|
260
259
|
- lib/cloud_powers/synapse/queue/poller.rb
|
261
|
-
- lib/cloud_powers/synapse/synapse.rb
|
262
260
|
- lib/cloud_powers/synapse/web_soc.rb
|
263
261
|
- lib/cloud_powers/synapse/web_soc/soc_client.rb
|
264
262
|
- lib/cloud_powers/synapse/web_soc/soc_server.rb
|
@@ -1,88 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
'Hey,'.kind_of? Object &&
|
5
|
-
'you guuyyys!'.is_a? String
|
6
|
-
#=> true
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
1.kind_of? Object &&
|
12
|
-
1.is_a? Numeric
|
13
|
-
#=> true
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
person_hash = {
|
19
|
-
name: 'Fredward Mc',
|
20
|
-
nationality: "'MERCA"
|
21
|
-
}
|
22
|
-
person_hash.kind_of? Object &&
|
23
|
-
person_hash.kind_of? Enumerable
|
24
|
-
# => true
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
class Person
|
30
|
-
self
|
31
|
-
end
|
32
|
-
|
33
|
-
fredward = Person.new
|
34
|
-
fredward.kind_of? Object &&
|
35
|
-
fredward.is_a? Person
|
36
|
-
#=> true, probably
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
system('compress', 'this.rb', 'and.sh', 'that.js', fredward, -> { some_method })
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
fredward
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
fredward.cats.select do |cat|
|
52
|
-
cat.pays_rent?
|
53
|
-
end
|
54
|
-
|
55
|
-
|
56
|
-
class Person
|
57
|
-
attr_accessor :name
|
58
|
-
|
59
|
-
def initialize(**opts)
|
60
|
-
@name = opts[:name]
|
61
|
-
end
|
62
|
-
|
63
|
-
def speak(words)
|
64
|
-
"Hi, I'm #{@name} and #{words}"
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
batman = Person.new(name: 'Bruce')
|
69
|
-
|
70
|
-
|
71
|
-
batman.speak("I'm Batman")
|
72
|
-
=> "Hi, I'm Bruce and I'm Batman"
|
73
|
-
|
74
|
-
|
75
|
-
class Yokl < Person
|
76
|
-
attr_accessor :cats
|
77
|
-
|
78
|
-
def initialize(**opts)
|
79
|
-
super
|
80
|
-
@cats = opts[:cats]
|
81
|
-
end
|
82
|
-
|
83
|
-
def speak(words)
|
84
|
-
"#{super(words).upcase}, YA'LL"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
fredward = Yokl.new(name: 'Fredward Mc', cats: ['bud', 'icehouse', 'miller', 'modelo', 'pabst'])
|
@@ -1,240 +0,0 @@
|
|
1
|
-
require 'aws-sdk'
|
2
|
-
Aws.use_bundled_cert!
|
3
|
-
require 'httparty'
|
4
|
-
require 'stubs/aws_stubs'
|
5
|
-
require_relative 'aws_resources'
|
6
|
-
require_relative 'helper'
|
7
|
-
require_relative './synapse/synapse'
|
8
|
-
require_relative 'zenv'
|
9
|
-
|
10
|
-
module Smash
|
11
|
-
module CloudPowers
|
12
|
-
module SelfAwareness
|
13
|
-
extend Smash::CloudPowers::Helper
|
14
|
-
extend Smash::CloudPowers::Synapse::Pipe
|
15
|
-
extend Smash::CloudPowers::Synapse::Queue
|
16
|
-
extend Smash::CloudPowers::Zenv
|
17
|
-
include Smash::CloudPowers::AwsResources
|
18
|
-
|
19
|
-
# Gets the instance time or the time it was called and as seconds from
|
20
|
-
# epoch
|
21
|
-
#
|
22
|
-
# Returns Integer
|
23
|
-
#
|
24
|
-
# Notes
|
25
|
-
# * TODO: use time codes
|
26
|
-
def boot_time
|
27
|
-
begin
|
28
|
-
@boot_time ||=
|
29
|
-
ec2.describe_instances(dry_run: zfind(:testing), instance_ids:[instance_id]).
|
30
|
-
reservations[0].instances[0].launch_time.to_i
|
31
|
-
rescue Aws::EC2::Errors::DryRunOperation
|
32
|
-
logger.info "dry run for testing: #{e}"
|
33
|
-
@boot_time ||= Time.now.to_i # comment the code below for development mode
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
# Send a status message on the status Pipe then terminates the instance.
|
38
|
-
def die!
|
39
|
-
Thread.kill(@status_thread) unless @status_thread.nil?
|
40
|
-
# blame = errors.sort_by(&:reverse).last.first
|
41
|
-
logger.info("The cause for the shutdown is TODO: fix SmashErrors")
|
42
|
-
|
43
|
-
pipe_to(:status_stream) do
|
44
|
-
{
|
45
|
-
instanceID: @instance_id,
|
46
|
-
type: 'status-update',
|
47
|
-
content: 'dying'
|
48
|
-
# extraInfo: blame
|
49
|
-
}
|
50
|
-
end
|
51
|
-
|
52
|
-
[:count, :wip].each do |queue|
|
53
|
-
delete_queue_message(queue, max_number_of_messages: 1)
|
54
|
-
end
|
55
|
-
send_logs_to_s3
|
56
|
-
begin
|
57
|
-
ec2.terminate_instances(dry_run: zfind('testing'), ids: [@instance_id])
|
58
|
-
rescue Aws::EC2::Error::DryRunOperation
|
59
|
-
logger.info "dry run testing in die! #{format_error_message(e)}"
|
60
|
-
@instance_id
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
# Get resource metadata, public host, boot time and task name
|
65
|
-
# and set them as instance variables
|
66
|
-
#
|
67
|
-
# Returns
|
68
|
-
# +Array+ of values, each from a separate key, set in the order
|
69
|
-
# in the Array
|
70
|
-
#
|
71
|
-
# Notes
|
72
|
-
# * See +#metadata_request()+
|
73
|
-
# * See +#attr_map!()+
|
74
|
-
def get_awareness!
|
75
|
-
keys = metadata_request
|
76
|
-
attr_map!(keys) { |key| metadata_request(key) }
|
77
|
-
boot_time # gets and sets @boot_time
|
78
|
-
task_name # gets and sets @task_name
|
79
|
-
instance_url # gets and sets @instance_url
|
80
|
-
end
|
81
|
-
|
82
|
-
# Assures there is always a valid instance id because many other Aws calls require it
|
83
|
-
#
|
84
|
-
# Returns
|
85
|
-
# +String+
|
86
|
-
def instance_id
|
87
|
-
@instance_id ||= metadata_request('instance_id')
|
88
|
-
end
|
89
|
-
|
90
|
-
# Gets and sets the public hostname of the instance
|
91
|
-
#
|
92
|
-
# Returns
|
93
|
-
# +String+ - the Public Hostname for this instance
|
94
|
-
#
|
95
|
-
# Notes
|
96
|
-
# * When this is being called from somewhere other than an Aws instance,
|
97
|
-
# a hardcoded example URL is returned because of the way instance metadata is retrieved
|
98
|
-
def instance_url
|
99
|
-
@instance_url ||= unless zfind('TESTING')
|
100
|
-
hostname_uri = 'http://169.254.169.254/latest/meta-data/public-hostname'
|
101
|
-
HTTParty.get(hostname_uri).parsed_response
|
102
|
-
else
|
103
|
-
'http://ec2-000-0-000-00.compute-0.amazonaws.com'
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
# Makes the http request to self/meta-data to get all the metadata keys or,
|
108
|
-
# if a key is given, the method makes the http request to get that
|
109
|
-
# particular key from the metadata
|
110
|
-
#
|
111
|
-
# Parameters
|
112
|
-
# * key +String+ (optional) (default is '') - the key for the metadata information
|
113
|
-
# you want from this instance.
|
114
|
-
#
|
115
|
-
# Returns
|
116
|
-
# * +Array+ if key is blank
|
117
|
-
# * +String+ if key is given
|
118
|
-
#
|
119
|
-
# Example
|
120
|
-
# metadata_request
|
121
|
-
# # => a +Hash+ containing every key => value pair AWS provides
|
122
|
-
# metadata_request('instance-id')
|
123
|
-
# # => 'abc-1234'
|
124
|
-
def metadata_request(key = '')
|
125
|
-
key = to_hyph(key)
|
126
|
-
begin
|
127
|
-
unless zfind('TESTING')
|
128
|
-
metadata_uri = "http://169.254.169.254/latest/meta-data/#{key}"
|
129
|
-
HTTParty.get(metadata_uri).parsed_response.split("\n")
|
130
|
-
else
|
131
|
-
stubbed_metadata = Smash::CloudPowers::AwsStubs.instance_metadata_stub
|
132
|
-
|
133
|
-
key.empty? ? stubbed_metadata.keys : stubbed_metadata[to_hyph(key)]
|
134
|
-
end
|
135
|
-
rescue Exception => e
|
136
|
-
logger.fatal format_error_message e
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
# Return the time since boot_time
|
141
|
-
#
|
142
|
-
# Returns
|
143
|
-
# +Integer+
|
144
|
-
#
|
145
|
-
# Notes
|
146
|
-
# * TODO: refactor to use valid time stamps for better tracking.
|
147
|
-
# * reason -> separate regions or OSs etc.
|
148
|
-
def run_time
|
149
|
-
Time.now.to_i - boot_time
|
150
|
-
end
|
151
|
-
|
152
|
-
# Send a message on a Pipe at an interval
|
153
|
-
#
|
154
|
-
# Parameters
|
155
|
-
# * opts +Hash+ (optional)
|
156
|
-
# * * +:interval+ - how long to wait between sending updates
|
157
|
-
# * * +:stream_name+ - name of stream you want to use
|
158
|
-
def send_frequent_status_updates(opts = {})
|
159
|
-
sleep_time = opts.delete(:interval) || 10
|
160
|
-
stream = opts.delete(:stream_name)
|
161
|
-
while true
|
162
|
-
message = lambda { |o| update_message_body(o.merge(content: status)) }
|
163
|
-
logger.info "Send update to status board #{message.call(opts)}"
|
164
|
-
pipe_to(stream || :status_stream) { message.call(opts) }
|
165
|
-
sleep sleep_time
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
# Get the instance status.
|
170
|
-
#
|
171
|
-
# Parameters
|
172
|
-
# * id +String+ (optional) (default is @instance_id)
|
173
|
-
#
|
174
|
-
# Returns String
|
175
|
-
def status(id = @instance_id)
|
176
|
-
begin
|
177
|
-
ec2.describe_instances(dry_run: zfind('TESTING'), instance_ids: [id]).
|
178
|
-
reservations[0].instances[0].state.name
|
179
|
-
rescue Aws::EC2::Errors::DryRunOperation
|
180
|
-
logger.info "Dry run flag set for testing: #{e}"
|
181
|
-
'testing'
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
|
186
|
-
# Search through tags on an instances using a regex pattern for your tag's
|
187
|
-
# key/name
|
188
|
-
#
|
189
|
-
# Parameters
|
190
|
-
# * pattern +String+|+Regex+
|
191
|
-
#
|
192
|
-
# Returns
|
193
|
-
# * +String+ - if a tag is found
|
194
|
-
# * +nil+ - if a tag is not found
|
195
|
-
def tag_search(pattern, id = @instance_id)
|
196
|
-
resp = ec2.describe_instances(instance_ids: [id].flatten).reservations.first
|
197
|
-
return nil if resp.nil?
|
198
|
-
resp.instances[0].tags.select do |tag|
|
199
|
-
tag.value if (tag.key =~ %r[#{pattern}])
|
200
|
-
end.collect.map(&:value).first
|
201
|
-
end
|
202
|
-
|
203
|
-
# Check self-tags for 'task' and act as an attr_accessor.
|
204
|
-
# A different node's tag's can be checked for a task by passing
|
205
|
-
# that instance's id as a parameter
|
206
|
-
#
|
207
|
-
# Parameters
|
208
|
-
# * id +String+ (optional) (default is @instance_id) - instance you want a tag from
|
209
|
-
#
|
210
|
-
# Returns
|
211
|
-
# +String+
|
212
|
-
#
|
213
|
-
# Notes
|
214
|
-
# * See <tt>tag_search()</tt>
|
215
|
-
def task_name(id = @instance_id)
|
216
|
-
# get @task_name
|
217
|
-
return @task_name unless @task_name.nil?
|
218
|
-
# set @task_name
|
219
|
-
# TODO: get all tasks instead of just the first
|
220
|
-
@task_name = tag_search('task', id)
|
221
|
-
end
|
222
|
-
|
223
|
-
# This method will return true if:
|
224
|
-
# * The run time is more than 5 minutes
|
225
|
-
# and
|
226
|
-
# * The run time is 5 minutes from the hour mark from when the instance started
|
227
|
-
# and will return false otherwise
|
228
|
-
#
|
229
|
-
# Returns
|
230
|
-
# +Boolean+
|
231
|
-
def time_is_up?
|
232
|
-
an_hours_time = 60 * 60
|
233
|
-
five_minutes_time = 60 * 5
|
234
|
-
|
235
|
-
return false if run_time < five_minutes_time
|
236
|
-
run_time % an_hours_time < five_minutes_time
|
237
|
-
end
|
238
|
-
end
|
239
|
-
end
|
240
|
-
end
|