cloud_powers 1.1.0 → 1.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/cloud_powers.gemspec +1 -1
- data/lib/cloud_powers.rb +4 -1
- data/lib/cloud_powers/helpers/logic_help.rb +26 -0
- data/lib/cloud_powers/helpers/path_help.rb +1 -1
- data/lib/cloud_powers/resource.rb +1 -0
- data/lib/cloud_powers/storage.rb +25 -25
- data/lib/cloud_powers/storage/bucket.rb +118 -46
- data/lib/cloud_powers/storage/local.rb +15 -2
- data/lib/cloud_powers/{synapse/synapse.rb → synapse.rb} +0 -0
- data/lib/cloud_powers/synapse/broadcast.rb +4 -1
- data/lib/cloud_powers/synapse/pipe.rb +23 -7
- data/lib/cloud_powers/synapse/queue.rb +18 -4
- data/lib/cloud_powers/synapse/queue/board.rb +1 -0
- data/lib/cloud_powers/version.rb +1 -1
- metadata +7 -9
- data/lib/cloud_powers/example_objects.rb +0 -88
- data/lib/cloud_powers/self_awareness.rb +0 -240
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 79e380ab917b71621e00f9877499d22ec6cda41a
|
4
|
+
data.tar.gz: ffe5f9828c11ddce25c1c5c7fcb91d70d4257dfa
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1ec413e2c2062102cd740c34c85524070bedb3b8fb19223065ae9927d2975293a2bb655f296677e03f8508f2645dce44387565c0d1c47a33167c3e9c4d12d82c
|
7
|
+
data.tar.gz: 36c347b259c84897e6cba930769cba4670cd41fcf4c32423608e922e27be67079ad6956b9355dce19fe31c0c36981716f0ef48a516b604ccebc7d4672b012f11
|
data/cloud_powers.gemspec
CHANGED
@@ -43,7 +43,7 @@ Gem::Specification.new do |spec|
|
|
43
43
|
spec.add_runtime_dependency 'zip-zip', '~> 0.3'
|
44
44
|
spec.add_runtime_dependency 'websocket-eventmachine-server', '~> 1.0.1'
|
45
45
|
spec.add_runtime_dependency 'websocket-eventmachine-client', '~> 1.2.0'
|
46
|
-
spec.add_runtime_dependency 'workflow'
|
46
|
+
spec.add_runtime_dependency 'workflow', '~> 1.2'
|
47
47
|
|
48
48
|
spec.add_development_dependency 'bundler', '~> 1.14.3'
|
49
49
|
spec.add_development_dependency 'byebug', '~> 9'
|
data/lib/cloud_powers.rb
CHANGED
@@ -4,8 +4,9 @@ require 'cloud_powers/helpers'
|
|
4
4
|
require 'cloud_powers/node'
|
5
5
|
require 'cloud_powers/resource'
|
6
6
|
require 'cloud_powers/storage'
|
7
|
+
require 'cloud_powers/synapse'
|
7
8
|
require 'cloud_powers/version'
|
8
|
-
require 'cloud_powers/
|
9
|
+
require 'cloud_powers/zenv'
|
9
10
|
|
10
11
|
# The Smash module allows us to use CloudPowers under a shared name space with other projects.
|
11
12
|
module Smash
|
@@ -21,5 +22,7 @@ module Smash
|
|
21
22
|
include Smash::CloudPowers::Synapse
|
22
23
|
# CRUD on Nodes, which are individual instances
|
23
24
|
include Smash::CloudPowers::Node
|
25
|
+
# Project information and tie-ins through dynamic lookups and caching
|
26
|
+
include Smash::CloudPowers::Zenv
|
24
27
|
end
|
25
28
|
end
|
@@ -1,3 +1,5 @@
|
|
1
|
+
require 'timeout'
|
2
|
+
|
1
3
|
module Smash
|
2
4
|
module CloudPowers
|
3
5
|
module LogicHelp
|
@@ -166,6 +168,30 @@ module Smash
|
|
166
168
|
extraInfo: updated_extra_info
|
167
169
|
}.merge(opts)
|
168
170
|
end
|
171
|
+
|
172
|
+
# Wait until a block finishes or a given time ellapses.
|
173
|
+
#
|
174
|
+
# Parameters
|
175
|
+
# * +wait_time+ +Integer+ - time to wait before returning
|
176
|
+
# * <tt>&block</tt> - block to execute. a <i>"truthy"</tt> result
|
177
|
+
# will finish this method
|
178
|
+
#
|
179
|
+
# Returns
|
180
|
+
# * +nil+ - if the time ellapses or your block returns nil
|
181
|
+
# * +Object+ - if your block succeeds, its return value is returned
|
182
|
+
#
|
183
|
+
# Example
|
184
|
+
# wait_until(3) { sleep 1; 'bla' }
|
185
|
+
# => 'bla' # returns after 1 second, not 3
|
186
|
+
#
|
187
|
+
# wait_until(1) { sleep 3; 'bla' }
|
188
|
+
# => nil # returns after 1 second because the block never finished
|
189
|
+
def wait_until(wait_time = 5)
|
190
|
+
begin
|
191
|
+
Timeout::timeout(wait_time) { block_given? ? (yield) : (sleep wait_time) }
|
192
|
+
rescue Timeout::Error => e
|
193
|
+
end
|
194
|
+
end
|
169
195
|
end
|
170
196
|
end
|
171
197
|
end
|
@@ -140,7 +140,7 @@ module Smash
|
|
140
140
|
|
141
141
|
def paths_gcd(*paths)
|
142
142
|
start_from = paths_lcd(*paths)
|
143
|
-
paths.sort.last.relative_path_from(start_from)
|
143
|
+
start_from + paths.sort.last.relative_path_from(start_from)
|
144
144
|
end
|
145
145
|
|
146
146
|
# Find all <i>directories</i> that match the given string.
|
data/lib/cloud_powers/storage.rb
CHANGED
@@ -113,21 +113,11 @@ module Smash
|
|
113
113
|
# | |_custom_greetings.js # could be an after effects JS script
|
114
114
|
def source_job(file)
|
115
115
|
storage_delegates(:bucket).map do |delegate|
|
116
|
-
|
117
|
-
|
118
|
-
|
116
|
+
# TODO!! Death to hardcoding
|
117
|
+
# Let's clean this up, eh?
|
118
|
+
loc = zfind('jobs') || 'jobs'
|
119
|
+
delegate.find(file, location: loc) { |f, data| f.write data }
|
119
120
|
end.flatten
|
120
|
-
# TODO: better path management
|
121
|
-
# bucket = zfind('jobs storage')
|
122
|
-
# if job_path(to_ruby_file_name(file)).nil?
|
123
|
-
# objects = s3.list_objects(bucket: bucket).contents.select do |f|
|
124
|
-
# /#{Regexp.escape file}/i =~ f.key
|
125
|
-
# end
|
126
|
-
|
127
|
-
# objects.each do |obj|
|
128
|
-
# s3.get_object(bucket: bucket, key: obj.key, response_target: job_path(file))
|
129
|
-
# end
|
130
|
-
# end
|
131
121
|
end
|
132
122
|
|
133
123
|
# Search through a bucket to find a file, based on a regex
|
@@ -165,6 +155,10 @@ module Smash
|
|
165
155
|
#
|
166
156
|
# Parameters
|
167
157
|
# * name +String+ - name of the storage you want to interact with
|
158
|
+
# * type +Symbol+ - the type of storage you want to build. The default
|
159
|
+
# is +:local+
|
160
|
+
# * +config+ +KeywordArgument+s - other configuration can be passed to
|
161
|
+
# customize the object just after initialization
|
168
162
|
#
|
169
163
|
# Returns
|
170
164
|
# Storage::[Local|Bucket]
|
@@ -176,8 +170,12 @@ module Smash
|
|
176
170
|
def build_storage(name:, type: nil, **config)
|
177
171
|
type = to_pascal(type || :local)
|
178
172
|
|
179
|
-
storage_resource = Smash::CloudPowers::Storage.const_get(type).
|
180
|
-
|
173
|
+
storage_resource = Smash::CloudPowers::Storage.const_get(type).build(
|
174
|
+
name: to_camel(name),
|
175
|
+
type: (type || :storage),
|
176
|
+
project_root: config[:project_root] || project_root,
|
177
|
+
**config
|
178
|
+
)
|
181
179
|
|
182
180
|
yield storage_resource if block_given?
|
183
181
|
|
@@ -206,12 +204,14 @@ module Smash
|
|
206
204
|
# get_queue_message_count
|
207
205
|
def create_storage(name:, type: nil, **config)
|
208
206
|
# default is :local
|
209
|
-
|
207
|
+
storage_resource = Smash::CloudPowers::Storage.const_get(type).create(
|
208
|
+
name: to_camel(name),
|
209
|
+
type: (type || :storage),
|
210
|
+
project_root: config[:project_root] || project_root,
|
211
|
+
**config
|
212
|
+
)
|
210
213
|
|
211
|
-
storage_resource
|
212
|
-
Smash::CloudPowers::Storage.const_get(to_pascal(type)).create!(
|
213
|
-
name: to_camel(name), **config
|
214
|
-
)
|
214
|
+
yield storage_resource if block_given?
|
215
215
|
|
216
216
|
attr_map(storage_resource.call_name => storage_resource) do |attribute, resource|
|
217
217
|
instance_attr_accessor attribute
|
@@ -334,11 +334,11 @@ module Smash
|
|
334
334
|
if respond_to? delegate_call_name
|
335
335
|
public_send delegate_call_name
|
336
336
|
else
|
337
|
-
config = storage_config(type).merge(
|
337
|
+
config = storage_config(type).merge(
|
338
|
+
{ name: 'delegate', type: type, project_root: project_root }
|
339
|
+
)
|
338
340
|
# set delegates up to search everywhere
|
339
|
-
build_storage(**config)
|
340
|
-
storage.tie_in_path = storage.origin
|
341
|
-
end
|
341
|
+
build_storage(**config)
|
342
342
|
end
|
343
343
|
end
|
344
344
|
end
|
@@ -7,16 +7,21 @@ module Smash
|
|
7
7
|
class Bucket < Smash::CloudPowers::Resource
|
8
8
|
include Smash::CloudPowers::Storage
|
9
9
|
|
10
|
-
attr_accessor :origin, :delimiter, :tie_in_path
|
10
|
+
attr_accessor :origin, :delimiter, :tie_in_path, :s3
|
11
11
|
|
12
12
|
def initialize(name:, client: s3, **config)
|
13
13
|
super
|
14
|
+
@s3 = client
|
14
15
|
@bucket = Aws::S3::Bucket.new(name: name, client: client)
|
15
|
-
|
16
|
+
@origin = config[:origin] || project_root.split.last
|
17
|
+
build_storage(
|
18
|
+
name: 'local',
|
19
|
+
named_type: 'storage',
|
20
|
+
project_root: @project_root
|
21
|
+
).link
|
16
22
|
@delimiter = config[:delimiter] || common_delimiter
|
17
23
|
# should be the 'lowest level' of storage this object is aware of
|
18
|
-
@
|
19
|
-
@tie_in_path = paths_gcd(local_storage.tie_in_path, project_root)
|
24
|
+
@tie_in_path = local_storage.tie_in_path
|
20
25
|
end
|
21
26
|
|
22
27
|
def self.tie_in_config
|
@@ -33,7 +38,9 @@ module Smash
|
|
33
38
|
# end
|
34
39
|
|
35
40
|
def create_resource
|
36
|
-
|
41
|
+
# TODO! Use Seahorse #on_complete, #on_success etc. callbacks
|
42
|
+
@response = s3.create_bucket(bucket: name).data
|
43
|
+
@remote_origin = @response.location.gsub(%r"^\/",'')
|
37
44
|
yield self if block_given?
|
38
45
|
self
|
39
46
|
end
|
@@ -43,6 +50,53 @@ module Smash
|
|
43
50
|
!!(@bucket.exists? rescue nil)
|
44
51
|
end
|
45
52
|
|
53
|
+
# Find the closest match to the request you're looking for with those
|
54
|
+
# params. If you pass this method a block, a file will be created and
|
55
|
+
# passed to the block, along with the data from the file that was found
|
56
|
+
# in the bucket.
|
57
|
+
#
|
58
|
+
# Parameters
|
59
|
+
# * +patterns+ +String+|+Regexp+ - names of files or buckets to look for
|
60
|
+
# * +location+ +String+ - narrow down the scope of the search
|
61
|
+
# * +opts+ +KeywordArgument+(s) - pass more specific configuration along
|
62
|
+
# to the search method(s) and object(s) used
|
63
|
+
#
|
64
|
+
# Returns
|
65
|
+
# * +String+:
|
66
|
+
# * if no +block+ was passed, only the name of the object found is
|
67
|
+
# returned
|
68
|
+
# * if a +block+ is passed to this method, you will have a +file+ and
|
69
|
+
# +data+ object to use as params. The file is created at the
|
70
|
+
# location of this +Storage+ object's +origin+, which is defaulted
|
71
|
+
# to +project_root+/zlib/+@name+_bucket. The +data+ object is the
|
72
|
+
# data from the object in the bucket
|
73
|
+
def find(*patterns, location: origin, **opts)
|
74
|
+
path = select(*patterns, location: location, **opts).first
|
75
|
+
file_name = path.kind_of?(Pathname) ? path.split.last.to_s : path
|
76
|
+
file_permissions = opts[:file_permissions] || 'a+'
|
77
|
+
path = local_storage.tie_in_path + path
|
78
|
+
|
79
|
+
if block_given?
|
80
|
+
begin
|
81
|
+
to_realpath(path)
|
82
|
+
File.open(path, file_permissions) do |file|
|
83
|
+
data = get_file(file_name, location: location).get.body.read
|
84
|
+
yield file, data
|
85
|
+
end
|
86
|
+
rescue Exception => e
|
87
|
+
byebug
|
88
|
+
if e =~ /no implicit conversion of nil into String/
|
89
|
+
logger.info "no file found from #{path} called #{file}"
|
90
|
+
nil
|
91
|
+
else
|
92
|
+
raise
|
93
|
+
end
|
94
|
+
end
|
95
|
+
else
|
96
|
+
path
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
46
100
|
# Find out if a job exists in the bucket
|
47
101
|
#
|
48
102
|
# Parameters
|
@@ -59,6 +113,7 @@ module Smash
|
|
59
113
|
|
60
114
|
def link
|
61
115
|
if exists?
|
116
|
+
byebug
|
62
117
|
# do stuff
|
63
118
|
else
|
64
119
|
save!
|
@@ -67,60 +122,74 @@ module Smash
|
|
67
122
|
@linked = @response.nil?
|
68
123
|
end
|
69
124
|
|
70
|
-
def select(*args, **opts)
|
71
|
-
|
72
|
-
|
125
|
+
# def select(*args, **opts)
|
126
|
+
# pattern = args.pop
|
127
|
+
# location = args.shift || origin
|
73
128
|
|
74
|
-
|
75
|
-
|
76
|
-
|
77
|
-
|
78
|
-
|
129
|
+
# names = if filename?(pattern)
|
130
|
+
# get_file_names(pattern, location: location)
|
131
|
+
# else
|
132
|
+
# get_bucket_names(pattern, location: location)
|
133
|
+
# end
|
79
134
|
|
80
135
|
|
81
|
-
|
82
|
-
|
83
|
-
|
84
|
-
|
85
|
-
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
91
|
-
|
92
|
-
|
93
|
-
end
|
136
|
+
# names = get_file_names(pattern, location: location)
|
137
|
+
# if block_given?
|
138
|
+
# names.map do |file_name|
|
139
|
+
# new_file = to_realpath(local_storage.find(file_name))
|
140
|
+
# File.open(new_file, 'r+') do |file|
|
141
|
+
# data = get_file(file_name, location: location).get.body.read
|
142
|
+
# yield file, data
|
143
|
+
# end
|
144
|
+
# end
|
145
|
+
# else
|
146
|
+
# names
|
147
|
+
# end
|
148
|
+
# end
|
149
|
+
def select(*patterns, location: @tie_in_path, **opts)
|
150
|
+
file_rights = opts[:file_rights] || 'a+'
|
151
|
+
names = patterns.map do |pattern|
|
152
|
+
pattern = pattern.kind_of?(Regexp) ? pattern : %r"#{pattern}"
|
94
153
|
|
95
|
-
|
96
|
-
|
97
|
-
|
98
|
-
|
99
|
-
|
154
|
+
# if filename?(pattern.source)
|
155
|
+
# get_file_names(pattern, location: location)
|
156
|
+
# else
|
157
|
+
# get_bucket_names(pattern, location: location)
|
158
|
+
# end
|
159
|
+
files_thread = Thread.new do
|
160
|
+
get_file_names(pattern, location: location)
|
161
|
+
end
|
162
|
+
bucket_thread = Thread.new do
|
163
|
+
get_bucket_names(pattern, location: location)
|
164
|
+
end
|
165
|
+
files_thread.join
|
166
|
+
bucket_thread.join
|
100
167
|
|
168
|
+
[files_thread.value, bucket_thread.value]
|
169
|
+
end.flatten
|
101
170
|
if block_given?
|
102
|
-
|
103
|
-
|
171
|
+
names.map do |file_name|
|
172
|
+
new_file = local_storage.touch(file_name)
|
173
|
+
File.open(new_file, file_rights) do |file|
|
104
174
|
data = get_file(file_name, location: location).get.body.read
|
105
175
|
yield file, data
|
106
176
|
end
|
107
|
-
rescue Exception => e
|
108
|
-
if e =~ /no implicit conversion of nil into String/
|
109
|
-
logger.info "no file found from #{path} called #{file}"
|
110
|
-
nil
|
111
|
-
else
|
112
|
-
super
|
113
|
-
end
|
114
177
|
end
|
115
178
|
else
|
116
|
-
|
179
|
+
names
|
117
180
|
end
|
118
181
|
end
|
119
182
|
|
120
183
|
# Notes:
|
121
|
-
# * See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3
|
122
|
-
|
123
|
-
|
184
|
+
# * See <i>https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/</i>
|
185
|
+
# <i>Client.html#put_object-instance_method</i>
|
186
|
+
def put(file_name, location = local_storage.tie_in_path, **config)
|
187
|
+
body = local_storage.find(
|
188
|
+
file_name,
|
189
|
+
location: location,
|
190
|
+
file_rights: 'r'
|
191
|
+
) { |data| data }
|
192
|
+
|
124
193
|
return if body.nil?
|
125
194
|
|
126
195
|
client.put_object(
|
@@ -129,6 +198,9 @@ module Smash
|
|
129
198
|
true
|
130
199
|
end
|
131
200
|
|
201
|
+
########################################################################
|
202
|
+
# PRIVATE METHODS
|
203
|
+
########################################################################
|
132
204
|
private
|
133
205
|
# Get a local file using the file name and optional location
|
134
206
|
#
|
@@ -138,7 +210,7 @@ module Smash
|
|
138
210
|
#
|
139
211
|
# Returns
|
140
212
|
# +String+|+IO+
|
141
|
-
def get_local_file_body(file, location =
|
213
|
+
def get_local_file_body(file, location = local_storage.tie_in_path)
|
142
214
|
scope = paths_gcd(tie_in_path, location)
|
143
215
|
local_storage.
|
144
216
|
find(file, location: scope, file_rights: 'r') { |data| data }
|
@@ -169,7 +241,7 @@ module Smash
|
|
169
241
|
end
|
170
242
|
|
171
243
|
def get_bucket_names(pattern, location: origin)
|
172
|
-
buckets = client.list_buckets.buckets.map(&:name)
|
244
|
+
buckets = client.list_buckets.buckets.map(&:name).grep(pattern)
|
173
245
|
end
|
174
246
|
|
175
247
|
# Get the actual object using S3::Bucket
|
@@ -11,7 +11,8 @@ module Smash
|
|
11
11
|
|
12
12
|
def initialize(name:, origin: zlib_home, **config)
|
13
13
|
super
|
14
|
-
|
14
|
+
tmp_origin = origin.kind_of?(Pathname) ? origin : to_pathname(origin)
|
15
|
+
@origin = paths_gcd(tmp_origin, project_root)
|
15
16
|
@tie_in_path = to_pathname(@origin, to_camel(name))
|
16
17
|
end
|
17
18
|
|
@@ -96,7 +97,6 @@ module Smash
|
|
96
97
|
grep(regex).
|
97
98
|
map { |path| to_realpath(path) }
|
98
99
|
end.flatten
|
99
|
-
|
100
100
|
if block_given?
|
101
101
|
paths.map { |path| yield File.open(path, file_rights) }
|
102
102
|
else
|
@@ -123,6 +123,19 @@ module Smash
|
|
123
123
|
path
|
124
124
|
end
|
125
125
|
end
|
126
|
+
|
127
|
+
def touch(path)
|
128
|
+
if @tie_in_path.nil?
|
129
|
+
logger.debug("Local#touch(#{path}) called")
|
130
|
+
super(path)
|
131
|
+
else
|
132
|
+
scope, last_piece = to_pathname(path).split
|
133
|
+
tied_in_path = paths_gcd(@tie_in_path, scope.expand_path) +
|
134
|
+
(filename?(last_piece) ? last_piece : '')
|
135
|
+
logger.debug("Local#touch(#{tied_in_path}) called")
|
136
|
+
super(tied_in_path)
|
137
|
+
end
|
138
|
+
end
|
126
139
|
end
|
127
140
|
end
|
128
141
|
end
|
File without changes
|
@@ -48,7 +48,10 @@ module Smash
|
|
48
48
|
def create_channel(name, **config)
|
49
49
|
channel_resource =
|
50
50
|
Smash::CloudPowers::Synapse::Broadcast::Channel.create!(
|
51
|
-
name: name,
|
51
|
+
name: name,
|
52
|
+
client: sns,
|
53
|
+
project_root: project_root,
|
54
|
+
**config
|
52
55
|
)
|
53
56
|
|
54
57
|
self.attr_map(channel_resource.call_name => channel_resource) do |attribute, resource|
|
@@ -8,21 +8,29 @@ module Smash
|
|
8
8
|
include Smash::CloudPowers::Helpers
|
9
9
|
include Smash::CloudPowers::Zenv
|
10
10
|
|
11
|
-
def build_pipe(name:, type: :stream, **config)
|
11
|
+
def build_pipe(name:, type: :stream, project_root: project_root, **config)
|
12
12
|
build_method_name = "build_#{type}"
|
13
13
|
if self.respond_to? build_method_name
|
14
|
-
self.public_send
|
14
|
+
self.public_send(build_method_name,
|
15
|
+
name: name,
|
16
|
+
project_root: project_root,
|
17
|
+
**config
|
18
|
+
)
|
15
19
|
else
|
16
|
-
build_stream(name: name, **config)
|
20
|
+
build_stream(name: name, project_root: project_root, **config)
|
17
21
|
end
|
18
22
|
end
|
19
23
|
|
20
24
|
def create_pipe(name:, type: :stream, **config)
|
21
25
|
create_method_name = "create_#{type}"
|
22
26
|
if self.respond_to? create_method_name
|
23
|
-
self.public_send
|
27
|
+
self.public_send(create_method_name,
|
28
|
+
name: name,
|
29
|
+
project_root: config[:project_root] || project_root,
|
30
|
+
**config
|
31
|
+
)
|
24
32
|
else
|
25
|
-
create_stream(name: name, **config)
|
33
|
+
create_stream(name: name, project_root: project_root, **config)
|
26
34
|
end
|
27
35
|
end
|
28
36
|
|
@@ -38,7 +46,10 @@ module Smash
|
|
38
46
|
# * returns false if the stream was not created
|
39
47
|
def build_stream(name:, client: kinesis, **config)
|
40
48
|
stream_resource = Smash::CloudPowers::Synapse::Pipe::Stream.build(
|
41
|
-
name: name,
|
49
|
+
name: name,
|
50
|
+
client: client,
|
51
|
+
project_root: config[:project_root] || project_root,
|
52
|
+
**config
|
42
53
|
)
|
43
54
|
|
44
55
|
self.attr_map(stream_resource.call_name => stream_resource) do |attribute, resource|
|
@@ -51,7 +62,12 @@ module Smash
|
|
51
62
|
|
52
63
|
def create_stream(name:, client: kinesis, **config)
|
53
64
|
stream_resource =
|
54
|
-
Smash::CloudPowers::Synapse::Pipe::Stream.create!(
|
65
|
+
Smash::CloudPowers::Synapse::Pipe::Stream.create!(
|
66
|
+
name: name,
|
67
|
+
client: client,
|
68
|
+
|
69
|
+
**config
|
70
|
+
)
|
55
71
|
|
56
72
|
self.attr_map(stream_resource.call_name => stream_resource) do |attribute, resource|
|
57
73
|
instance_attr_accessor attribute
|
@@ -112,7 +112,11 @@ module Smash
|
|
112
112
|
def build_queue(name:, type: :board, **config)
|
113
113
|
build_method_name = "build_#{type}"
|
114
114
|
if self.respond_to? build_method_name
|
115
|
-
self.public_send
|
115
|
+
self.public_send(build_method_name,
|
116
|
+
name: name,
|
117
|
+
project_root: config[:project_root] || project_root,
|
118
|
+
**config
|
119
|
+
)
|
116
120
|
else
|
117
121
|
build_board(name: name, **config)
|
118
122
|
end
|
@@ -149,7 +153,11 @@ module Smash
|
|
149
153
|
def create_queue(name:, type: :board, **config)
|
150
154
|
create_method_name = "build_#{type}"
|
151
155
|
if self.respond_to? create_method_name
|
152
|
-
self.public_send
|
156
|
+
self.public_send(create_method_name,
|
157
|
+
name: name,
|
158
|
+
project_root: config[:project_root] || project_root,
|
159
|
+
**config
|
160
|
+
)
|
153
161
|
else
|
154
162
|
create_queue(name: name, **config)
|
155
163
|
end
|
@@ -171,7 +179,10 @@ module Smash
|
|
171
179
|
# => https://sqs.us-west-2.amazonaws.com/81234567/exampleQueue
|
172
180
|
def build_board(name:, client: sqs, **config)
|
173
181
|
board_resource = Smash::CloudPowers::Synapse::Queue::Board.build(
|
174
|
-
name: to_camel(name),
|
182
|
+
name: to_camel(name),
|
183
|
+
client: client,
|
184
|
+
project_root: config[:project_root] || project_root,
|
185
|
+
**config
|
175
186
|
)
|
176
187
|
|
177
188
|
attr_map(board_resource.call_name => board_resource) do |attribute, resource|
|
@@ -195,7 +206,10 @@ module Smash
|
|
195
206
|
# get_queue_message_count
|
196
207
|
def create_board(name:, client: sqs, **config)
|
197
208
|
board_resource = Smash::CloudPowers::Synapse::Queue::Board.create!(
|
198
|
-
name: to_camel(name),
|
209
|
+
name: to_camel(name),
|
210
|
+
client: sqs,
|
211
|
+
project_root: config[:project_root] || project_root,
|
212
|
+
**config
|
199
213
|
)
|
200
214
|
|
201
215
|
attr_map(board_resource.call_name => board_resource) do |attribute, resource|
|
data/lib/cloud_powers/version.rb
CHANGED
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloud_powers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.1.
|
4
|
+
version: 1.1.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Phillipps
|
@@ -10,7 +10,7 @@ authors:
|
|
10
10
|
autorequire:
|
11
11
|
bindir: exe
|
12
12
|
cert_chain: []
|
13
|
-
date: 2017-03-
|
13
|
+
date: 2017-03-27 00:00:00.000000000 Z
|
14
14
|
dependencies:
|
15
15
|
- !ruby/object:Gem::Dependency
|
16
16
|
name: activesupport-core-ext
|
@@ -128,16 +128,16 @@ dependencies:
|
|
128
128
|
name: workflow
|
129
129
|
requirement: !ruby/object:Gem::Requirement
|
130
130
|
requirements:
|
131
|
-
- - "
|
131
|
+
- - "~>"
|
132
132
|
- !ruby/object:Gem::Version
|
133
|
-
version: '
|
133
|
+
version: '1.2'
|
134
134
|
type: :runtime
|
135
135
|
prerelease: false
|
136
136
|
version_requirements: !ruby/object:Gem::Requirement
|
137
137
|
requirements:
|
138
|
-
- - "
|
138
|
+
- - "~>"
|
139
139
|
- !ruby/object:Gem::Version
|
140
|
-
version: '
|
140
|
+
version: '1.2'
|
141
141
|
- !ruby/object:Gem::Dependency
|
142
142
|
name: bundler
|
143
143
|
requirement: !ruby/object:Gem::Requirement
|
@@ -236,7 +236,6 @@ files:
|
|
236
236
|
- lib/cloud_powers/auth.rb
|
237
237
|
- lib/cloud_powers/aws_resources.rb
|
238
238
|
- lib/cloud_powers/creatable.rb
|
239
|
-
- lib/cloud_powers/example_objects.rb
|
240
239
|
- lib/cloud_powers/helpers.rb
|
241
240
|
- lib/cloud_powers/helpers/lang_help.rb
|
242
241
|
- lib/cloud_powers/helpers/logic_help.rb
|
@@ -244,12 +243,12 @@ files:
|
|
244
243
|
- lib/cloud_powers/node.rb
|
245
244
|
- lib/cloud_powers/node/instance.rb
|
246
245
|
- lib/cloud_powers/resource.rb
|
247
|
-
- lib/cloud_powers/self_awareness.rb
|
248
246
|
- lib/cloud_powers/smash_error.rb
|
249
247
|
- lib/cloud_powers/storage.rb
|
250
248
|
- lib/cloud_powers/storage/bucket.rb
|
251
249
|
- lib/cloud_powers/storage/local.rb
|
252
250
|
- lib/cloud_powers/stubs/aws_stubs.rb
|
251
|
+
- lib/cloud_powers/synapse.rb
|
253
252
|
- lib/cloud_powers/synapse/broadcast.rb
|
254
253
|
- lib/cloud_powers/synapse/broadcast/channel.rb
|
255
254
|
- lib/cloud_powers/synapse/pipe.rb
|
@@ -258,7 +257,6 @@ files:
|
|
258
257
|
- lib/cloud_powers/synapse/queue.rb
|
259
258
|
- lib/cloud_powers/synapse/queue/board.rb
|
260
259
|
- lib/cloud_powers/synapse/queue/poller.rb
|
261
|
-
- lib/cloud_powers/synapse/synapse.rb
|
262
260
|
- lib/cloud_powers/synapse/web_soc.rb
|
263
261
|
- lib/cloud_powers/synapse/web_soc/soc_client.rb
|
264
262
|
- lib/cloud_powers/synapse/web_soc/soc_server.rb
|
@@ -1,88 +0,0 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
'Hey,'.kind_of? Object &&
|
5
|
-
'you guuyyys!'.is_a? String
|
6
|
-
#=> true
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
1.kind_of? Object &&
|
12
|
-
1.is_a? Numeric
|
13
|
-
#=> true
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
person_hash = {
|
19
|
-
name: 'Fredward Mc',
|
20
|
-
nationality: "'MERCA"
|
21
|
-
}
|
22
|
-
person_hash.kind_of? Object &&
|
23
|
-
person_hash.kind_of? Enumerable
|
24
|
-
# => true
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
class Person
|
30
|
-
self
|
31
|
-
end
|
32
|
-
|
33
|
-
fredward = Person.new
|
34
|
-
fredward.kind_of? Object &&
|
35
|
-
fredward.is_a? Person
|
36
|
-
#=> true, probably
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
system('compress', 'this.rb', 'and.sh', 'that.js', fredward, -> { some_method })
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
fredward
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
fredward.cats.select do |cat|
|
52
|
-
cat.pays_rent?
|
53
|
-
end
|
54
|
-
|
55
|
-
|
56
|
-
class Person
|
57
|
-
attr_accessor :name
|
58
|
-
|
59
|
-
def initialize(**opts)
|
60
|
-
@name = opts[:name]
|
61
|
-
end
|
62
|
-
|
63
|
-
def speak(words)
|
64
|
-
"Hi, I'm #{@name} and #{words}"
|
65
|
-
end
|
66
|
-
end
|
67
|
-
|
68
|
-
batman = Person.new(name: 'Bruce')
|
69
|
-
|
70
|
-
|
71
|
-
batman.speak("I'm Batman")
|
72
|
-
=> "Hi, I'm Bruce and I'm Batman"
|
73
|
-
|
74
|
-
|
75
|
-
class Yokl < Person
|
76
|
-
attr_accessor :cats
|
77
|
-
|
78
|
-
def initialize(**opts)
|
79
|
-
super
|
80
|
-
@cats = opts[:cats]
|
81
|
-
end
|
82
|
-
|
83
|
-
def speak(words)
|
84
|
-
"#{super(words).upcase}, YA'LL"
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
fredward = Yokl.new(name: 'Fredward Mc', cats: ['bud', 'icehouse', 'miller', 'modelo', 'pabst'])
|
@@ -1,240 +0,0 @@
|
|
1
|
-
require 'aws-sdk'
|
2
|
-
Aws.use_bundled_cert!
|
3
|
-
require 'httparty'
|
4
|
-
require 'stubs/aws_stubs'
|
5
|
-
require_relative 'aws_resources'
|
6
|
-
require_relative 'helper'
|
7
|
-
require_relative './synapse/synapse'
|
8
|
-
require_relative 'zenv'
|
9
|
-
|
10
|
-
module Smash
|
11
|
-
module CloudPowers
|
12
|
-
module SelfAwareness
|
13
|
-
extend Smash::CloudPowers::Helper
|
14
|
-
extend Smash::CloudPowers::Synapse::Pipe
|
15
|
-
extend Smash::CloudPowers::Synapse::Queue
|
16
|
-
extend Smash::CloudPowers::Zenv
|
17
|
-
include Smash::CloudPowers::AwsResources
|
18
|
-
|
19
|
-
# Gets the instance time or the time it was called and as seconds from
|
20
|
-
# epoch
|
21
|
-
#
|
22
|
-
# Returns Integer
|
23
|
-
#
|
24
|
-
# Notes
|
25
|
-
# * TODO: use time codes
|
26
|
-
def boot_time
|
27
|
-
begin
|
28
|
-
@boot_time ||=
|
29
|
-
ec2.describe_instances(dry_run: zfind(:testing), instance_ids:[instance_id]).
|
30
|
-
reservations[0].instances[0].launch_time.to_i
|
31
|
-
rescue Aws::EC2::Errors::DryRunOperation
|
32
|
-
logger.info "dry run for testing: #{e}"
|
33
|
-
@boot_time ||= Time.now.to_i # comment the code below for development mode
|
34
|
-
end
|
35
|
-
end
|
36
|
-
|
37
|
-
# Send a status message on the status Pipe then terminates the instance.
|
38
|
-
def die!
|
39
|
-
Thread.kill(@status_thread) unless @status_thread.nil?
|
40
|
-
# blame = errors.sort_by(&:reverse).last.first
|
41
|
-
logger.info("The cause for the shutdown is TODO: fix SmashErrors")
|
42
|
-
|
43
|
-
pipe_to(:status_stream) do
|
44
|
-
{
|
45
|
-
instanceID: @instance_id,
|
46
|
-
type: 'status-update',
|
47
|
-
content: 'dying'
|
48
|
-
# extraInfo: blame
|
49
|
-
}
|
50
|
-
end
|
51
|
-
|
52
|
-
[:count, :wip].each do |queue|
|
53
|
-
delete_queue_message(queue, max_number_of_messages: 1)
|
54
|
-
end
|
55
|
-
send_logs_to_s3
|
56
|
-
begin
|
57
|
-
ec2.terminate_instances(dry_run: zfind('testing'), ids: [@instance_id])
|
58
|
-
rescue Aws::EC2::Error::DryRunOperation
|
59
|
-
logger.info "dry run testing in die! #{format_error_message(e)}"
|
60
|
-
@instance_id
|
61
|
-
end
|
62
|
-
end
|
63
|
-
|
64
|
-
# Get resource metadata, public host, boot time and task name
|
65
|
-
# and set them as instance variables
|
66
|
-
#
|
67
|
-
# Returns
|
68
|
-
# +Array+ of values, each from a separate key, set in the order
|
69
|
-
# in the Array
|
70
|
-
#
|
71
|
-
# Notes
|
72
|
-
# * See +#metadata_request()+
|
73
|
-
# * See +#attr_map!()+
|
74
|
-
def get_awareness!
|
75
|
-
keys = metadata_request
|
76
|
-
attr_map!(keys) { |key| metadata_request(key) }
|
77
|
-
boot_time # gets and sets @boot_time
|
78
|
-
task_name # gets and sets @task_name
|
79
|
-
instance_url # gets and sets @instance_url
|
80
|
-
end
|
81
|
-
|
82
|
-
# Assures there is always a valid instance id because many other Aws calls require it
|
83
|
-
#
|
84
|
-
# Returns
|
85
|
-
# +String+
|
86
|
-
def instance_id
|
87
|
-
@instance_id ||= metadata_request('instance_id')
|
88
|
-
end
|
89
|
-
|
90
|
-
# Gets and sets the public hostname of the instance
|
91
|
-
#
|
92
|
-
# Returns
|
93
|
-
# +String+ - the Public Hostname for this instance
|
94
|
-
#
|
95
|
-
# Notes
|
96
|
-
# * When this is being called from somewhere other than an Aws instance,
|
97
|
-
# a hardcoded example URL is returned because of the way instance metadata is retrieved
|
98
|
-
def instance_url
|
99
|
-
@instance_url ||= unless zfind('TESTING')
|
100
|
-
hostname_uri = 'http://169.254.169.254/latest/meta-data/public-hostname'
|
101
|
-
HTTParty.get(hostname_uri).parsed_response
|
102
|
-
else
|
103
|
-
'http://ec2-000-0-000-00.compute-0.amazonaws.com'
|
104
|
-
end
|
105
|
-
end
|
106
|
-
|
107
|
-
# Makes the http request to self/meta-data to get all the metadata keys or,
|
108
|
-
# if a key is given, the method makes the http request to get that
|
109
|
-
# particular key from the metadata
|
110
|
-
#
|
111
|
-
# Parameters
|
112
|
-
# * key +String+ (optional) (default is '') - the key for the metadata information
|
113
|
-
# you want from this instance.
|
114
|
-
#
|
115
|
-
# Returns
|
116
|
-
# * +Array+ if key is blank
|
117
|
-
# * +String+ if key is given
|
118
|
-
#
|
119
|
-
# Example
|
120
|
-
# metadata_request
|
121
|
-
# # => a +Hash+ containing every key => value pair AWS provides
|
122
|
-
# metadata_request('instance-id')
|
123
|
-
# # => 'abc-1234'
|
124
|
-
def metadata_request(key = '')
|
125
|
-
key = to_hyph(key)
|
126
|
-
begin
|
127
|
-
unless zfind('TESTING')
|
128
|
-
metadata_uri = "http://169.254.169.254/latest/meta-data/#{key}"
|
129
|
-
HTTParty.get(metadata_uri).parsed_response.split("\n")
|
130
|
-
else
|
131
|
-
stubbed_metadata = Smash::CloudPowers::AwsStubs.instance_metadata_stub
|
132
|
-
|
133
|
-
key.empty? ? stubbed_metadata.keys : stubbed_metadata[to_hyph(key)]
|
134
|
-
end
|
135
|
-
rescue Exception => e
|
136
|
-
logger.fatal format_error_message e
|
137
|
-
end
|
138
|
-
end
|
139
|
-
|
140
|
-
# Return the time since boot_time
|
141
|
-
#
|
142
|
-
# Returns
|
143
|
-
# +Integer+
|
144
|
-
#
|
145
|
-
# Notes
|
146
|
-
# * TODO: refactor to use valid time stamps for better tracking.
|
147
|
-
# * reason -> separate regions or OSs etc.
|
148
|
-
def run_time
|
149
|
-
Time.now.to_i - boot_time
|
150
|
-
end
|
151
|
-
|
152
|
-
# Send a message on a Pipe at an interval
|
153
|
-
#
|
154
|
-
# Parameters
|
155
|
-
# * opts +Hash+ (optional)
|
156
|
-
# * * +:interval+ - how long to wait between sending updates
|
157
|
-
# * * +:stream_name+ - name of stream you want to use
|
158
|
-
def send_frequent_status_updates(opts = {})
|
159
|
-
sleep_time = opts.delete(:interval) || 10
|
160
|
-
stream = opts.delete(:stream_name)
|
161
|
-
while true
|
162
|
-
message = lambda { |o| update_message_body(o.merge(content: status)) }
|
163
|
-
logger.info "Send update to status board #{message.call(opts)}"
|
164
|
-
pipe_to(stream || :status_stream) { message.call(opts) }
|
165
|
-
sleep sleep_time
|
166
|
-
end
|
167
|
-
end
|
168
|
-
|
169
|
-
# Get the instance status.
|
170
|
-
#
|
171
|
-
# Parameters
|
172
|
-
# * id +String+ (optional) (default is @instance_id)
|
173
|
-
#
|
174
|
-
# Returns String
|
175
|
-
def status(id = @instance_id)
|
176
|
-
begin
|
177
|
-
ec2.describe_instances(dry_run: zfind('TESTING'), instance_ids: [id]).
|
178
|
-
reservations[0].instances[0].state.name
|
179
|
-
rescue Aws::EC2::Errors::DryRunOperation
|
180
|
-
logger.info "Dry run flag set for testing: #{e}"
|
181
|
-
'testing'
|
182
|
-
end
|
183
|
-
end
|
184
|
-
|
185
|
-
|
186
|
-
# Search through tags on an instances using a regex pattern for your tag's
|
187
|
-
# key/name
|
188
|
-
#
|
189
|
-
# Parameters
|
190
|
-
# * pattern +String+|+Regex+
|
191
|
-
#
|
192
|
-
# Returns
|
193
|
-
# * +String+ - if a tag is found
|
194
|
-
# * +nil+ - if a tag is not found
|
195
|
-
def tag_search(pattern, id = @instance_id)
|
196
|
-
resp = ec2.describe_instances(instance_ids: [id].flatten).reservations.first
|
197
|
-
return nil if resp.nil?
|
198
|
-
resp.instances[0].tags.select do |tag|
|
199
|
-
tag.value if (tag.key =~ %r[#{pattern}])
|
200
|
-
end.collect.map(&:value).first
|
201
|
-
end
|
202
|
-
|
203
|
-
# Check self-tags for 'task' and act as an attr_accessor.
|
204
|
-
# A different node's tag's can be checked for a task by passing
|
205
|
-
# that instance's id as a parameter
|
206
|
-
#
|
207
|
-
# Parameters
|
208
|
-
# * id +String+ (optional) (default is @instance_id) - instance you want a tag from
|
209
|
-
#
|
210
|
-
# Returns
|
211
|
-
# +String+
|
212
|
-
#
|
213
|
-
# Notes
|
214
|
-
# * See <tt>tag_search()</tt>
|
215
|
-
def task_name(id = @instance_id)
|
216
|
-
# get @task_name
|
217
|
-
return @task_name unless @task_name.nil?
|
218
|
-
# set @task_name
|
219
|
-
# TODO: get all tasks instead of just the first
|
220
|
-
@task_name = tag_search('task', id)
|
221
|
-
end
|
222
|
-
|
223
|
-
# This method will return true if:
|
224
|
-
# * The run time is more than 5 minutes
|
225
|
-
# and
|
226
|
-
# * The run time is 5 minutes from the hour mark from when the instance started
|
227
|
-
# and will return false otherwise
|
228
|
-
#
|
229
|
-
# Returns
|
230
|
-
# +Boolean+
|
231
|
-
def time_is_up?
|
232
|
-
an_hours_time = 60 * 60
|
233
|
-
five_minutes_time = 60 * 5
|
234
|
-
|
235
|
-
return false if run_time < five_minutes_time
|
236
|
-
run_time % an_hours_time < five_minutes_time
|
237
|
-
end
|
238
|
-
end
|
239
|
-
end
|
240
|
-
end
|