cloud_powers 0.2.5 → 0.2.6
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/Gemfile +0 -1
- data/Gemfile.lock +22 -9
- data/Gemfile.lock_b +59 -0
- data/cloud_powers.gemspec +10 -6
- data/lib/cloud_powers/auth.rb +1 -0
- data/lib/cloud_powers/aws_resources.rb +4 -4
- data/lib/cloud_powers/context.rb +8 -6
- data/lib/cloud_powers/helper.rb +2 -9
- data/lib/cloud_powers/self_awareness.rb +9 -5
- data/lib/cloud_powers/synapse/pipe/pipe.rb +51 -5
- data/lib/cloud_powers/synapse/queue/board.rb +31 -19
- data/lib/cloud_powers/synapse/queue/queue.rb +35 -30
- data/lib/cloud_powers/synapse/synapse.rb +5 -0
- data/lib/cloud_powers/synapse/websocket/websocclient.rb +37 -0
- data/lib/cloud_powers/synapse/websocket/websocserver.rb +43 -0
- data/lib/cloud_powers/version.rb +1 -1
- data/lib/cloud_powers.rb +1 -0
- metadata +43 -7
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: d02a1e6e07212386a014c0c8855065e29b6be6ea
|
4
|
+
data.tar.gz: 317ca03ffc825961ff980d7d7b25824b13da97b7
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 977411458b0021e684834bdf33791b6e39c6e84beebd41760d6a85c8d7451a84a6dd61dc7839f19c1322c415a64e9fa55f801eb660357f061f6c4160ffa279ca
|
7
|
+
data.tar.gz: 2d2e4a52a63d0e3bb52f9f6e5838dd0bfc336952929c6d7333bb3c87d46b9700402f7da5c5e20ab775314938685b70ba8acde7ff0e3fb2b6a919b3a2402fde26
|
data/Gemfile
CHANGED
data/Gemfile.lock
CHANGED
@@ -1,27 +1,30 @@
|
|
1
1
|
PATH
|
2
2
|
remote: .
|
3
3
|
specs:
|
4
|
-
cloud_powers (0.2.
|
4
|
+
cloud_powers (0.2.6)
|
5
5
|
activesupport-core-ext (~> 4)
|
6
6
|
aws-sdk (~> 2)
|
7
7
|
dotenv (~> 2.1)
|
8
8
|
httparty (~> 0.14)
|
9
9
|
rubyzip (~> 1.2)
|
10
|
+
websocket-eventmachine-client
|
11
|
+
websocket-eventmachine-server
|
10
12
|
zip-zip (~> 0.3)
|
11
13
|
|
12
14
|
GEM
|
13
15
|
remote: https://rubygems.org/
|
14
16
|
specs:
|
15
17
|
activesupport-core-ext (4.0.0.3)
|
16
|
-
aws-sdk (2.6.
|
17
|
-
aws-sdk-resources (= 2.6.
|
18
|
-
aws-sdk-core (2.6.
|
18
|
+
aws-sdk (2.6.6)
|
19
|
+
aws-sdk-resources (= 2.6.6)
|
20
|
+
aws-sdk-core (2.6.6)
|
19
21
|
jmespath (~> 1.0)
|
20
|
-
aws-sdk-resources (2.6.
|
21
|
-
aws-sdk-core (= 2.6.
|
22
|
-
byebug (9.0.
|
22
|
+
aws-sdk-resources (2.6.6)
|
23
|
+
aws-sdk-core (= 2.6.6)
|
24
|
+
byebug (9.0.6)
|
23
25
|
diff-lcs (1.2.5)
|
24
26
|
dotenv (2.1.1)
|
27
|
+
eventmachine (1.2.0.1)
|
25
28
|
httparty (0.14.0)
|
26
29
|
multi_xml (>= 0.5.2)
|
27
30
|
jmespath (1.3.1)
|
@@ -31,7 +34,7 @@ GEM
|
|
31
34
|
rspec-core (~> 3.5.0)
|
32
35
|
rspec-expectations (~> 3.5.0)
|
33
36
|
rspec-mocks (~> 3.5.0)
|
34
|
-
rspec-core (3.5.
|
37
|
+
rspec-core (3.5.4)
|
35
38
|
rspec-support (~> 3.5.0)
|
36
39
|
rspec-expectations (3.5.0)
|
37
40
|
diff-lcs (>= 1.2.0, < 2.0)
|
@@ -41,6 +44,16 @@ GEM
|
|
41
44
|
rspec-support (~> 3.5.0)
|
42
45
|
rspec-support (3.5.0)
|
43
46
|
rubyzip (1.2.0)
|
47
|
+
websocket (1.2.3)
|
48
|
+
websocket-eventmachine-base (1.2.0)
|
49
|
+
eventmachine (~> 1.0)
|
50
|
+
websocket (~> 1.0)
|
51
|
+
websocket-native (~> 1.0)
|
52
|
+
websocket-eventmachine-client (1.2.0)
|
53
|
+
websocket-eventmachine-base (~> 1.0)
|
54
|
+
websocket-eventmachine-server (1.0.1)
|
55
|
+
websocket-eventmachine-base (~> 1.0)
|
56
|
+
websocket-native (1.0.0)
|
44
57
|
zip-zip (0.3)
|
45
58
|
rubyzip (>= 1.0.0)
|
46
59
|
|
@@ -55,4 +68,4 @@ DEPENDENCIES
|
|
55
68
|
rspec (~> 3.0)
|
56
69
|
|
57
70
|
BUNDLED WITH
|
58
|
-
1.
|
71
|
+
1.13.2
|
data/Gemfile.lock_b
ADDED
@@ -0,0 +1,59 @@
|
|
1
|
+
PATH
|
2
|
+
remote: .
|
3
|
+
specs:
|
4
|
+
cloud_powers (0.2.4)
|
5
|
+
activesupport-core-ext (~> 4)
|
6
|
+
aws-sdk (~> 2)
|
7
|
+
dotenv (~> 2.1)
|
8
|
+
httparty (~> 0.14)
|
9
|
+
rubyzip (~> 1.2)
|
10
|
+
zip-zip (~> 0.3)
|
11
|
+
|
12
|
+
GEM
|
13
|
+
remote: https://rubygems.org/
|
14
|
+
specs:
|
15
|
+
activesupport-core-ext (4.0.0.3)
|
16
|
+
aws-sdk (2.6.3)
|
17
|
+
aws-sdk-resources (= 2.6.3)
|
18
|
+
aws-sdk-core (2.6.3)
|
19
|
+
jmespath (~> 1.0)
|
20
|
+
aws-sdk-resources (2.6.3)
|
21
|
+
aws-sdk-core (= 2.6.3)
|
22
|
+
byebug (9.0.5)
|
23
|
+
diff-lcs (1.2.5)
|
24
|
+
dotenv (2.1.1)
|
25
|
+
httparty (0.14.0)
|
26
|
+
multi_xml (>= 0.5.2)
|
27
|
+
jmespath (1.3.1)
|
28
|
+
multi_xml (0.5.5)
|
29
|
+
rake (10.5.0)
|
30
|
+
rspec (3.5.0)
|
31
|
+
rspec-core (~> 3.5.0)
|
32
|
+
rspec-expectations (~> 3.5.0)
|
33
|
+
rspec-mocks (~> 3.5.0)
|
34
|
+
rspec-core (3.5.3)
|
35
|
+
rspec-support (~> 3.5.0)
|
36
|
+
rspec-expectations (3.5.0)
|
37
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
38
|
+
rspec-support (~> 3.5.0)
|
39
|
+
rspec-mocks (3.5.0)
|
40
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
41
|
+
rspec-support (~> 3.5.0)
|
42
|
+
rspec-support (3.5.0)
|
43
|
+
rubyzip (1.2.0)
|
44
|
+
zip-zip (0.3)
|
45
|
+
rubyzip (>= 1.0.0)
|
46
|
+
websocket (>= 1.2.3)
|
47
|
+
|
48
|
+
PLATFORMS
|
49
|
+
ruby
|
50
|
+
|
51
|
+
DEPENDENCIES
|
52
|
+
bundler (~> 1.12)
|
53
|
+
byebug (~> 9)
|
54
|
+
cloud_powers!
|
55
|
+
rake (~> 10.0)
|
56
|
+
rspec (~> 3.0)
|
57
|
+
|
58
|
+
BUNDLED WITH
|
59
|
+
1.12.5
|
data/cloud_powers.gemspec
CHANGED
@@ -7,19 +7,21 @@ Gem::Specification.new do |spec|
|
|
7
7
|
spec.required_ruby_version = '~> 2.3.0'
|
8
8
|
spec.name = 'cloud_powers'
|
9
9
|
spec.version = CloudPowers::VERSION
|
10
|
-
spec.
|
11
|
-
spec.email = 'adam.phillipps@gmail.com'
|
10
|
+
spec.authors = ['Adam Phillipps', 'Faisal Irfan']
|
11
|
+
spec.email = ['adam.phillipps@gmail.com', 'Faisal@smashanalytics.com']
|
12
12
|
spec.summary = %q{Cloud providers wrapper. Currently only AWS is supported.}
|
13
13
|
spec.description = <<-EOF
|
14
14
|
CloudPowers is a wrapper around AWS and in the future, other cloud service Providers.
|
15
15
|
It was developed specifically for the Brain project but hopefully can be used
|
16
16
|
in any other ruby project that needs to use cloud service providers' resources.
|
17
|
-
Version 0.2.
|
18
|
-
features you can find in the docs.
|
19
|
-
|
17
|
+
Version 0.2.6 has a little EC2, S3, SQS, SNS, Kinesis, websockets and a few other
|
18
|
+
features you can find in the docs. There is also limitted support for stubbing
|
19
|
+
AWS RESTful API calls. That can come in handy for local testing and extra setup on
|
20
|
+
AWS resource clients.
|
21
|
+
The next versions will have more Kinesis, Workflow integration and IoT.
|
20
22
|
This project is actively being developed, so more additions, specs and docs
|
21
23
|
will be added and updated frequently with new funcionality but the gem will
|
22
|
-
follow good practices for versioning.
|
24
|
+
follow good practices for versioning. Input is always welcome.
|
23
25
|
Enjoy!
|
24
26
|
EOF
|
25
27
|
spec.homepage = 'https://github.com/adam-phillipps/cloud_powers'
|
@@ -36,6 +38,8 @@ Gem::Specification.new do |spec|
|
|
36
38
|
spec.add_runtime_dependency 'httparty', '~> 0.14'
|
37
39
|
spec.add_runtime_dependency 'rubyzip', '~> 1.2'
|
38
40
|
spec.add_runtime_dependency 'zip-zip', '~> 0.3'
|
41
|
+
spec.add_runtime_dependency 'websocket-eventmachine-server'
|
42
|
+
spec.add_runtime_dependency 'websocket-eventmachine-client'
|
39
43
|
|
40
44
|
spec.add_development_dependency 'bundler', '~> 1.12'
|
41
45
|
spec.add_development_dependency 'byebug', '~> 9'
|
data/lib/cloud_powers/auth.rb
CHANGED
@@ -9,6 +9,7 @@ module Smash
|
|
9
9
|
|
10
10
|
# This method is able to be called before an object is instantiated in order
|
11
11
|
# to provide a region in AWS-land.
|
12
|
+
# === @returns: The region set in configuration or a 'us-west-2' default String
|
12
13
|
def self.region
|
13
14
|
zfind(:aws_region) || 'us-west-2'
|
14
15
|
end
|
@@ -67,10 +67,10 @@ module Smash
|
|
67
67
|
end
|
68
68
|
|
69
69
|
def sqs(opts = {})
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
)
|
70
|
+
config = {
|
71
|
+
credentials: Auth.creds
|
72
|
+
}.merge(opts)
|
73
|
+
@sqs ||= Aws::SQS::Client.new(config)
|
74
74
|
end
|
75
75
|
end
|
76
76
|
end
|
data/lib/cloud_powers/context.rb
CHANGED
@@ -91,7 +91,7 @@ module Smash
|
|
91
91
|
# `<Array|List <Enumerable>`
|
92
92
|
# e.g.
|
93
93
|
# flat
|
94
|
-
# ```
|
94
|
+
# ```Ruby
|
95
95
|
# [
|
96
96
|
# object_name_1, config_1a, config_2a, ...,
|
97
97
|
# object_2, config_1b, etc,
|
@@ -100,16 +100,16 @@ module Smash
|
|
100
100
|
# ```
|
101
101
|
# grouped
|
102
102
|
# or
|
103
|
-
# ```
|
103
|
+
# ```Ruby
|
104
104
|
# [
|
105
105
|
# [object_name_1, config_1a, config_2a, ...],
|
106
106
|
# [object_2, config_1b, etc],
|
107
107
|
# ...
|
108
108
|
# ]
|
109
109
|
# ```
|
110
|
-
# structured
|
111
110
|
# or
|
112
|
-
#
|
111
|
+
# structured
|
112
|
+
# ```Ruby
|
113
113
|
# [
|
114
114
|
# [object_name_1, [config_1a, config_2a, ...]],
|
115
115
|
# [object_2, [config_1b, etc]],
|
@@ -117,7 +117,7 @@ module Smash
|
|
117
117
|
# ]
|
118
118
|
# ```
|
119
119
|
# returns
|
120
|
-
# ```
|
120
|
+
# ```Ruby
|
121
121
|
# {
|
122
122
|
# object_1: [config_1a, config_2a, ...],
|
123
123
|
# object_2: [config_1b, ...],
|
@@ -133,7 +133,7 @@ module Smash
|
|
133
133
|
# Translates an Array into a valid @structure Hash
|
134
134
|
# === @param arr <Array>
|
135
135
|
# e.g.
|
136
|
-
# ```
|
136
|
+
# ```Ruby
|
137
137
|
# [:task, 'demo', :queue, 'name1', {other config hash},...,:pipe, 'name1']
|
138
138
|
# ```
|
139
139
|
# === @returns Hash
|
@@ -190,6 +190,8 @@ module Smash
|
|
190
190
|
# class _should_ exist in. It can be a vanilla version, where the @structure
|
191
191
|
# is a Hash, structured correctly or it can be serialized into JSON or it can
|
192
192
|
# be an Array
|
193
|
+
# === @params: args String
|
194
|
+
# === @returns: Boolean
|
193
195
|
def valid_args?(args)
|
194
196
|
case args
|
195
197
|
when Hash
|
data/lib/cloud_powers/helper.rb
CHANGED
@@ -88,9 +88,7 @@ module Smash
|
|
88
88
|
key = yield k
|
89
89
|
|
90
90
|
value = if v.kind_of?(Hash)
|
91
|
-
deep_modify_keys_with(v)
|
92
|
-
Proc.new.call(new_key)
|
93
|
-
end
|
91
|
+
deep_modify_keys_with(v) { |new_key| Proc.new.call(new_key) }
|
94
92
|
else
|
95
93
|
v
|
96
94
|
end
|
@@ -108,11 +106,6 @@ module Smash
|
|
108
106
|
end
|
109
107
|
end
|
110
108
|
|
111
|
-
def errors
|
112
|
-
# TODO: wow...needs work
|
113
|
-
$errors ||= SmashError.instance
|
114
|
-
end
|
115
|
-
|
116
109
|
# Join the message and backtrace into a String with line breaks
|
117
110
|
def format_error_message(error)
|
118
111
|
begin
|
@@ -129,7 +122,7 @@ module Smash
|
|
129
122
|
@log_file ||= zfind('LOG_FILE')
|
130
123
|
end
|
131
124
|
|
132
|
-
# @returns: An instance of Logger, cached as @logger
|
125
|
+
# === @returns: An instance of Logger, cached as @logger
|
133
126
|
def logger
|
134
127
|
@logger ||= create_logger
|
135
128
|
end
|
@@ -74,12 +74,16 @@ module Smash
|
|
74
74
|
end
|
75
75
|
|
76
76
|
# Gets and sets the public hostname of the instance
|
77
|
+
# === @returns String
|
78
|
+
# === Notes:
|
79
|
+
# * When this is being called from somewhere other than an Aws instance, a hardcoded example URL is returned
|
80
|
+
# because of the way instance metadata is retrieved
|
77
81
|
def instance_url
|
78
|
-
@instance_url ||=
|
79
|
-
'
|
80
|
-
else
|
81
|
-
hostname_uri = 'http://169.254.169.254/latest/meta-data/instance-id'
|
82
|
+
@instance_url ||= unless zfind('TESTING')
|
83
|
+
hostname_uri = 'http://169.254.169.254/latest/meta-data/public-hostname'
|
82
84
|
HTTParty.get(hostname_uri).parsed_response
|
85
|
+
else
|
86
|
+
'http://ec2-000-0-000-00.compute-0.amazonaws.com'
|
83
87
|
end
|
84
88
|
end
|
85
89
|
|
@@ -95,7 +99,7 @@ module Smash
|
|
95
99
|
metadata_uri = "http://169.254.169.254/latest/meta-data/#{key}"
|
96
100
|
HTTParty.get(metadata_uri).parsed_response.split("\n")
|
97
101
|
else
|
98
|
-
require_relative
|
102
|
+
require_relative "#{Smash::CloudPowers::CLOUDPOWERS_SPECS_ROOT}/stubs/aws_stubs"
|
99
103
|
stubbed_metadata = Smash::CloudPowers::AwsStubs::INSTANCE_METADATA_STUB
|
100
104
|
|
101
105
|
key.empty? ? stubbed_metadata.keys : stubbed_metadata[to_hyph(key)]
|
@@ -6,6 +6,12 @@ module Smash
|
|
6
6
|
include Smash::CloudPowers::Helper
|
7
7
|
|
8
8
|
module Pipe
|
9
|
+
# Create a Kinesis stream or wait until the stream with the given name is
|
10
|
+
# through being created.
|
11
|
+
# === @params: name String
|
12
|
+
# === @returns: Boolean or nil
|
13
|
+
# * returns true or false if the request was successful
|
14
|
+
# * returns false
|
9
15
|
def create_stream(name)
|
10
16
|
begin
|
11
17
|
config = stream_config(stream_name: env(name))
|
@@ -15,13 +21,11 @@ module Smash
|
|
15
21
|
rescue Exception => e
|
16
22
|
if e.kind_of? Aws::Kinesis::Errors::ResourceInUseException
|
17
23
|
logger.info "#{name} already created"
|
18
|
-
|
19
|
-
return if stream_status == 'ACTIVE'
|
24
|
+
return if stream_status(name) == 'ACTIVE'
|
20
25
|
logger.info "Not ready for traffic. Wait for 30 seconds..."
|
21
|
-
sleep
|
26
|
+
sleep 1
|
22
27
|
nil # no request -> no response
|
23
28
|
else
|
24
|
-
# TODO: make the errors thing work
|
25
29
|
error_message = format_error_message(e)
|
26
30
|
logger.error error_message
|
27
31
|
false # the request was not successful
|
@@ -29,11 +33,14 @@ module Smash
|
|
29
33
|
end
|
30
34
|
end
|
31
35
|
|
36
|
+
# Use the KCL and LangDaemon to read from a stream
|
37
|
+
# === @params: stream String
|
32
38
|
def flow_from_pipe(stream)
|
33
39
|
throw NotImplementedError
|
34
40
|
end
|
35
41
|
|
36
42
|
def flow_to_pipe(stream)
|
43
|
+
throw NotImplementedError
|
37
44
|
create_stream(stream) unless stream_exists? stream
|
38
45
|
records = yield if block_given?
|
39
46
|
body = message_body_collection(records)
|
@@ -44,8 +51,10 @@ module Smash
|
|
44
51
|
# TODO: what to return? true?
|
45
52
|
end
|
46
53
|
|
54
|
+
# Read messages from the Pipe without using the KCL
|
55
|
+
# === @params: stream String
|
47
56
|
def from_pipe(stream)
|
48
|
-
#
|
57
|
+
# implement get_records and/or other consuming app stuff
|
49
58
|
throw NotImplementedError
|
50
59
|
end
|
51
60
|
|
@@ -53,6 +62,15 @@ module Smash
|
|
53
62
|
throw NotImplementedError
|
54
63
|
end
|
55
64
|
|
65
|
+
# Default message package. This method yields the basic configuration
|
66
|
+
# and message body for a stream and all options can be changed.
|
67
|
+
# === @params: opts Hash (optional)
|
68
|
+
# * stream_name: String name of the stream to pipe to
|
69
|
+
# * data: String message to send
|
70
|
+
# * partition_key: String defaults to @instance_id
|
71
|
+
# === @returns: Hash
|
72
|
+
# === Notes:
|
73
|
+
# See #instance_id()
|
56
74
|
def pipe_message_body(opts = {})
|
57
75
|
{
|
58
76
|
stream_name: env(opts[:stream_name]) || env('status_stream'),
|
@@ -61,7 +79,20 @@ module Smash
|
|
61
79
|
}
|
62
80
|
end
|
63
81
|
|
82
|
+
# Use Kinesis streams to send a message. The message is given to the method
|
83
|
+
# through a block that gets passed to the method.
|
84
|
+
# === @params: stream String
|
85
|
+
# === block: a block that generates a string that will be used in the message body
|
86
|
+
# === @returns: the sequence_number from the sent message.
|
87
|
+
# === Example use
|
88
|
+
# ```Ruby
|
89
|
+
# pipe_to(:status_stream) do
|
90
|
+
# # the return from the inner method is what is sent
|
91
|
+
# do_some_stuff_to_generate_a_message()
|
92
|
+
# end
|
93
|
+
# ```
|
64
94
|
def pipe_to(stream)
|
95
|
+
message = ''
|
65
96
|
create_stream(stream) unless stream_exists? stream
|
66
97
|
message = yield if block_given?
|
67
98
|
body = update_message_body(message)
|
@@ -70,6 +101,9 @@ module Smash
|
|
70
101
|
@last_sequence_number = resp.sequence_number
|
71
102
|
end
|
72
103
|
|
104
|
+
# New stream config with sensible defaults
|
105
|
+
# === @params: opts Hash
|
106
|
+
# * stream_name:
|
73
107
|
def stream_config(opts = {})
|
74
108
|
config = {
|
75
109
|
stream_name: opts[:stream_name] || env('status_stream'),
|
@@ -77,13 +111,25 @@ module Smash
|
|
77
111
|
}
|
78
112
|
end
|
79
113
|
|
114
|
+
# Find out if the stream already exists.
|
115
|
+
# === @params: name String
|
116
|
+
# === @returns: Boolean
|
80
117
|
def stream_exists?(name)
|
81
118
|
begin
|
82
119
|
kinesis.describe_stream(stream_name: env(name))
|
120
|
+
true
|
83
121
|
rescue Aws::Kinesis::Errors::ResourceNotFoundException => e
|
84
122
|
false
|
85
123
|
end
|
86
124
|
end
|
125
|
+
|
126
|
+
# Get the status name for this stream
|
127
|
+
# === @params: name String
|
128
|
+
# === @returns: stream status, one of:
|
129
|
+
# CREATING, DELETING, ACTIVE, UPDATING
|
130
|
+
def stream_status(name)
|
131
|
+
kinesis.describe_stream(name).stream_description.stream_status
|
132
|
+
end
|
87
133
|
end
|
88
134
|
end
|
89
135
|
end
|
@@ -10,7 +10,7 @@ module Smash
|
|
10
10
|
include Smash::CloudPowers::Helper
|
11
11
|
include Smash::CloudPowers::Zenv
|
12
12
|
|
13
|
-
attr_accessor :address, :name, :poller
|
13
|
+
attr_accessor :address, :name, :poller, :sqs
|
14
14
|
|
15
15
|
# Takes a `name` and creates a Board object.
|
16
16
|
# The #new method is wrapped in #build() and #create!() but isn't labeled private so
|
@@ -18,24 +18,29 @@ module Smash
|
|
18
18
|
# The Board doesn't create Queue(s) or any other API calls until it is instructed to.
|
19
19
|
# @params: name <String>: the name of the board can be used to find its arn/url etc
|
20
20
|
# @returns: Queue::Board
|
21
|
-
# not
|
22
|
-
def initialize(name)
|
21
|
+
# not `#new()`
|
22
|
+
def initialize(name, this_sqs)# = sqs)
|
23
|
+
@sqs = this_sqs
|
23
24
|
@name = name
|
24
25
|
end
|
25
26
|
|
26
27
|
# Gives back a string representation of the instance variable for this board.
|
27
|
-
# @returns:
|
28
|
-
#
|
28
|
+
# === @returns: String
|
29
|
+
# *the instance variable for this Board in string format
|
30
|
+
# === Example:
|
31
|
+
# ```Ruby
|
29
32
|
# board = Board.new(:backlog)
|
30
33
|
# Smash.instance_variable_get(board.i_var)
|
31
34
|
# #=> Board <name: :backlog ...>
|
35
|
+
# ```
|
32
36
|
def i_var
|
33
37
|
"@#{@name}"
|
34
38
|
end
|
35
39
|
|
36
40
|
# Gives the Queue address (URL). First the environment is searched, using Zenv and if nothing is
|
37
41
|
# found, the best guess attempt at the correct address is used.
|
38
|
-
# @returns:
|
42
|
+
# === @returns:
|
43
|
+
# * queue address <String>
|
39
44
|
def address
|
40
45
|
zfind(@name) || best_guess_address
|
41
46
|
end
|
@@ -44,27 +49,28 @@ module Smash
|
|
44
49
|
# to build a standard URL for SQS. The only problem with using this last resort is you may need
|
45
50
|
# to use a Queue from a different region, account or name but it can be a handy catch-all for the URLs
|
46
51
|
# for most cases.
|
52
|
+
# === @returns String
|
53
|
+
# * exp. "https://sqs.us-west-2.amazonaws.com/12345678/fooBar"
|
47
54
|
def best_guess_address
|
48
55
|
"https://sqs.#{zfind(:aws_region)}.amazonaws.com/#{zfind(:account_number)}/#{@name}"
|
49
56
|
end
|
50
57
|
|
51
58
|
# Builds a Queue::Board object and returns it. No API calls are sent using this method
|
52
|
-
# @params: name <String>
|
53
|
-
# @returns: Queue::Board
|
54
|
-
def self.build(name)
|
55
|
-
new(name)
|
59
|
+
# === @params: name <String>
|
60
|
+
# === @returns: Queue::Board
|
61
|
+
def self.build(name, this_sqs)#)
|
62
|
+
new(name, this_sqs)
|
56
63
|
end
|
57
64
|
|
58
65
|
# Builds then Creates the Object and makes the API call to SQS to create the queue
|
59
|
-
# @params: name <String>
|
60
|
-
# @returns: Queue::Board with an actual queue in SQS
|
61
|
-
def self.create!(name)
|
62
|
-
|
63
|
-
built_board.create_queue!
|
66
|
+
# === @params: name <String>
|
67
|
+
# === @returns: Queue::Board with an actual queue in SQS
|
68
|
+
def self.create!(name, this_sqs)# = nil)
|
69
|
+
build(name, this_sqs).create_queue!
|
64
70
|
end
|
65
71
|
|
66
72
|
# Creates an actual Queue in SQS using the standard format for a queue name (camel case)
|
67
|
-
# @returns: Queue::Board
|
73
|
+
# === @returns: Queue::Board
|
68
74
|
def create_queue!
|
69
75
|
begin
|
70
76
|
sqs.create_queue(queue_name: to_camel(@name))
|
@@ -91,8 +97,12 @@ module Smash
|
|
91
97
|
end
|
92
98
|
|
93
99
|
# Gets a QueuePoller for the Queue attached to this Board instance
|
100
|
+
# === @returns Aws::SQS::QueuePoller
|
94
101
|
def poller
|
95
|
-
|
102
|
+
# Provide an existing SQS Client if one exists.
|
103
|
+
# This sets up stubbing and other stuff for later
|
104
|
+
args = @sqs.nil? ? address : [address, { client: sqs }]
|
105
|
+
@poller ||= Aws::SQS::QueuePoller.new(*args)
|
96
106
|
end
|
97
107
|
|
98
108
|
# Retrieves a message from the Queue and deletes it from the Queue in SQS
|
@@ -108,10 +118,12 @@ module Smash
|
|
108
118
|
end
|
109
119
|
|
110
120
|
# Sends the given message to the queue
|
111
|
-
# @params: message, which is either used as JSON or converted into it
|
121
|
+
# === @params: message, which is either used as JSON or converted into it
|
112
122
|
def send_message(message)
|
113
123
|
send_queue_message(
|
114
|
-
address,
|
124
|
+
address,
|
125
|
+
(valid_json?(message) ? message : message.to_json),
|
126
|
+
sqs
|
115
127
|
)
|
116
128
|
end
|
117
129
|
end
|
@@ -5,8 +5,8 @@ module Smash
|
|
5
5
|
module CloudPowers
|
6
6
|
module Synapse
|
7
7
|
module Queue
|
8
|
-
include Smash::CloudPowers::Helper
|
9
8
|
include Smash::CloudPowers::AwsResources
|
9
|
+
include Smash::CloudPowers::Helper
|
10
10
|
|
11
11
|
# A simpl Struct that acts as a Name to URL map
|
12
12
|
NUMap = Struct.new(:set_name, :set_url) do
|
@@ -21,31 +21,38 @@ module Smash
|
|
21
21
|
|
22
22
|
# This method can be used to parse a queue name from its address. It can be handy if you need the name
|
23
23
|
# of a queue but you don't want the overhead of creating a Board object.
|
24
|
+
# === @params: url String
|
25
|
+
# === @returns: String
|
26
|
+
# === Example:
|
27
|
+
# ```Ruby
|
28
|
+
# board_name('https://sqs.us-west-53.amazonaws.com/001101010010/fooBar')
|
29
|
+
# => fooBar
|
30
|
+
# ```
|
24
31
|
def board_name(url)
|
25
32
|
url.to_s.split('/').last
|
26
33
|
end
|
27
34
|
|
35
|
+
# This method builds a Queue::Board object for you to use but doesn't
|
36
|
+
# invoke the #create! method, so no API call is made to create the queue
|
37
|
+
# on SQS. This can be used if the board already exists.
|
38
|
+
# === @params: name <String>: name of the queue you want to interact with
|
39
|
+
# === @returns: Queue::Board
|
40
|
+
def build_queue(name)
|
41
|
+
Smash::CloudPowers::Queue::Board.build(to_camel(name), sqs)
|
42
|
+
end
|
43
|
+
|
28
44
|
# This method allows you to create a queue on SQS without explicitly creating a Board object
|
29
45
|
# === @params: name <String>: The name of the queue to be created
|
30
46
|
# === @returns: Queue::Board
|
31
47
|
def create_queue!(name)
|
32
48
|
begin
|
33
|
-
Smash::CloudPowers::Queue::Board.create!(to_camel(name))
|
49
|
+
Smash::CloudPowers::Queue::Board.create!(to_camel(name), sqs)
|
34
50
|
rescue Aws::SQS::Errors::QueueDeletedRecently => e
|
35
51
|
sleep 5
|
36
52
|
retry
|
37
53
|
end
|
38
54
|
end
|
39
55
|
|
40
|
-
# This method builds a Queue::Board object for you to use but doesn't
|
41
|
-
# invoke the #create! method, so no API call is made to create the queue
|
42
|
-
# on SQS. This can be used if the board already exists.
|
43
|
-
# === @params: name <String>: name of the queue you want to interact with
|
44
|
-
# === @returns: Queue::Board
|
45
|
-
def build_queue(name)
|
46
|
-
Smash::CloudPowers::Queue::Board.build(to_camel(name))
|
47
|
-
end
|
48
|
-
|
49
56
|
# Deletes a queue message without caring about reading/interacting with the message.
|
50
57
|
# This is usually used for progress tracking, ie; a Neuron takes a message from the Backlog, moves it to
|
51
58
|
# WIP and deletes it from Backlog. Then repeats these steps for the remaining States in the Workflow
|
@@ -81,12 +88,12 @@ module Smash
|
|
81
88
|
# Polls the given board with the given options hash and a block that interacts with
|
82
89
|
# the message that is retrieved from the queue
|
83
90
|
# === @params: board <String>[, opts <Hash>]
|
84
|
-
# board is the name of the queue you want to poll
|
85
|
-
# opts can have any AWS::SQS polling option
|
86
|
-
#
|
91
|
+
# * `board` is the name of the queue you want to poll
|
92
|
+
# * `opts` can have any AWS::SQS polling option
|
93
|
+
# * `&block` is the block that is used to interact with the message that was retrieved
|
87
94
|
# === @returns the results from the `message` and the `block` that interacts with the message(s)
|
88
|
-
def poll(
|
89
|
-
this_poller = queue_poller(
|
95
|
+
def poll(board_name, opts = {})
|
96
|
+
this_poller = queue_poller(board_name)
|
90
97
|
results = nil
|
91
98
|
this_poller.poll(opts) do |msg|
|
92
99
|
results = yield msg, this_poller if block_given?
|
@@ -102,25 +109,26 @@ module Smash
|
|
102
109
|
# === @params: board_name <String>: name of the Queue you want to gain a QueuePoller for
|
103
110
|
# === @returns: @<board_name:Queue::Board>
|
104
111
|
def queue_poller(board_name)
|
105
|
-
board = Smash::CloudPowers::Queue::Board.create!(board_name)
|
112
|
+
board = Smash::CloudPowers::Queue::Board.create!(board_name, sqs)
|
106
113
|
|
107
114
|
unless instance_variable_defined?(board.i_var)
|
108
|
-
instance_variable_set(
|
109
|
-
board.i_var,
|
110
|
-
board
|
111
|
-
)
|
115
|
+
instance_variable_set(board.i_var, board)
|
112
116
|
end
|
113
117
|
instance_variable_get(board.i_var).poller
|
114
118
|
end
|
115
119
|
|
116
|
-
# Checks SQS for the existence of this queue
|
120
|
+
# Checks SQS for the existence of this queue using the #queue_search() method
|
121
|
+
# === @params: name String
|
122
|
+
# === @returns: Boolean
|
123
|
+
# === Notes:
|
124
|
+
# * see #queue_search()
|
117
125
|
def queue_exists?(name)
|
118
|
-
!
|
126
|
+
!queue_search(name).empty?
|
119
127
|
end
|
120
128
|
|
121
129
|
# Searches for a queue based on the name
|
122
|
-
# === @params: name
|
123
|
-
# === @returns: queue_urls
|
130
|
+
# === @params: name String
|
131
|
+
# === @returns: queue_urls [String]
|
124
132
|
def queue_search(name)
|
125
133
|
sqs.list_queues(queue_name_prefix: name).queue_urls
|
126
134
|
end
|
@@ -128,11 +136,8 @@ module Smash
|
|
128
136
|
# Sends a given message to a given queue
|
129
137
|
# === @params: address <String>: address of the queue you want to interact with
|
130
138
|
# === @returns: queue_urls <Array<String>> # TODO: verify this. maybe it was late...
|
131
|
-
def send_queue_message(address, message)
|
132
|
-
|
133
|
-
queue_url: address,
|
134
|
-
message_body: message
|
135
|
-
)
|
139
|
+
def send_queue_message(address, message, this_sqs = sqs)
|
140
|
+
this_sqs.send_message(queue_url: address, message_body: message)
|
136
141
|
end
|
137
142
|
end
|
138
143
|
end
|
@@ -2,6 +2,8 @@ require_relative './broadcast/broadcast'
|
|
2
2
|
require_relative './queue/board'
|
3
3
|
require_relative './queue/queue'
|
4
4
|
require_relative './pipe/pipe'
|
5
|
+
require_relative './websocket/websocserver'
|
6
|
+
require_relative './websocket/websocclient'
|
5
7
|
|
6
8
|
module Smash
|
7
9
|
module CloudPowers
|
@@ -11,10 +13,13 @@ module Smash
|
|
11
13
|
# or loaded, logging info and any other high-throughput/data-centric application with
|
12
14
|
# - Queue is a module that is primarily used for asynchronous communications between a sender
|
13
15
|
# and any number of users or apps that _might_ need to use it
|
16
|
+
# - WebSocServer ...._Faisal's turn_...
|
14
17
|
module Synapse
|
15
18
|
include Smash::CloudPowers::Synapse::Broadcast
|
16
19
|
include Smash::CloudPowers::Synapse::Pipe
|
17
20
|
include Smash::CloudPowers::Synapse::Queue
|
21
|
+
include Smash::CloudPowers::Synapse::WebSocClient
|
22
|
+
include Smash::CloudPowers::Synapse::WebSocServer
|
18
23
|
end
|
19
24
|
end
|
20
25
|
end
|
@@ -0,0 +1,37 @@
|
|
1
|
+
require 'websocket-eventmachine-client'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module Synapse
|
6
|
+
module WebSocClient
|
7
|
+
def create_websoc_client( host, port )
|
8
|
+
EM.run do
|
9
|
+
|
10
|
+
ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://' + host + ':' + port)
|
11
|
+
|
12
|
+
ws.onopen do
|
13
|
+
puts "Connected"
|
14
|
+
end
|
15
|
+
|
16
|
+
ws.onmessage do |msg, type|
|
17
|
+
puts "Received message: #{msg}"
|
18
|
+
end
|
19
|
+
|
20
|
+
ws.onerror do |error|
|
21
|
+
puts "Error ==> #{error}"
|
22
|
+
end
|
23
|
+
|
24
|
+
ws.onclose do |code, reason|
|
25
|
+
puts "Disconnected with status code: #{code}"
|
26
|
+
puts "Disconnected with status message: #{reason}"
|
27
|
+
end
|
28
|
+
|
29
|
+
EventMachine.next_tick do
|
30
|
+
ws.send "Hello Server!"
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
@@ -0,0 +1,43 @@
|
|
1
|
+
require 'websocket-eventmachine-server'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module Synapse
|
6
|
+
module WebSocServer
|
7
|
+
def create_websoc_server( host, port )
|
8
|
+
@channel = EM::Channel.new
|
9
|
+
Thread.new do
|
10
|
+
EM.run do
|
11
|
+
WebSocket::EventMachine::Server.start(:host => host, :port => port) do |ws|
|
12
|
+
sid = nil
|
13
|
+
|
14
|
+
ws.onopen do
|
15
|
+
puts "Client connected"
|
16
|
+
sid = @channel.subscribe { |msg| ws.send msg }
|
17
|
+
end
|
18
|
+
|
19
|
+
ws.onmessage do |msg, type|
|
20
|
+
puts "Received message: #{msg}"
|
21
|
+
@channel.push "<#{sid}>: #{msg}"
|
22
|
+
end
|
23
|
+
|
24
|
+
ws.onerror do |error|
|
25
|
+
puts "Error occured: #{error}"
|
26
|
+
end
|
27
|
+
|
28
|
+
ws.onclose do
|
29
|
+
puts "Client disconnected"
|
30
|
+
@channel.unsubscribe(sid) unless @channel.nil?
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def send( msg )
|
38
|
+
@channel.push "#{msg}" unless @channel.nil? nil
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
data/lib/cloud_powers/version.rb
CHANGED
data/lib/cloud_powers.rb
CHANGED
metadata
CHANGED
@@ -1,14 +1,15 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: cloud_powers
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.6
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Adam Phillipps
|
8
|
+
- Faisal Irfan
|
8
9
|
autorequire:
|
9
10
|
bindir: exe
|
10
11
|
cert_chain: []
|
11
|
-
date: 2016-10-
|
12
|
+
date: 2016-10-10 00:00:00.000000000 Z
|
12
13
|
dependencies:
|
13
14
|
- !ruby/object:Gem::Dependency
|
14
15
|
name: activesupport-core-ext
|
@@ -94,6 +95,34 @@ dependencies:
|
|
94
95
|
- - "~>"
|
95
96
|
- !ruby/object:Gem::Version
|
96
97
|
version: '0.3'
|
98
|
+
- !ruby/object:Gem::Dependency
|
99
|
+
name: websocket-eventmachine-server
|
100
|
+
requirement: !ruby/object:Gem::Requirement
|
101
|
+
requirements:
|
102
|
+
- - ">="
|
103
|
+
- !ruby/object:Gem::Version
|
104
|
+
version: '0'
|
105
|
+
type: :runtime
|
106
|
+
prerelease: false
|
107
|
+
version_requirements: !ruby/object:Gem::Requirement
|
108
|
+
requirements:
|
109
|
+
- - ">="
|
110
|
+
- !ruby/object:Gem::Version
|
111
|
+
version: '0'
|
112
|
+
- !ruby/object:Gem::Dependency
|
113
|
+
name: websocket-eventmachine-client
|
114
|
+
requirement: !ruby/object:Gem::Requirement
|
115
|
+
requirements:
|
116
|
+
- - ">="
|
117
|
+
- !ruby/object:Gem::Version
|
118
|
+
version: '0'
|
119
|
+
type: :runtime
|
120
|
+
prerelease: false
|
121
|
+
version_requirements: !ruby/object:Gem::Requirement
|
122
|
+
requirements:
|
123
|
+
- - ">="
|
124
|
+
- !ruby/object:Gem::Version
|
125
|
+
version: '0'
|
97
126
|
- !ruby/object:Gem::Dependency
|
98
127
|
name: bundler
|
99
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -154,14 +183,18 @@ description: |2
|
|
154
183
|
CloudPowers is a wrapper around AWS and in the future, other cloud service Providers.
|
155
184
|
It was developed specifically for the Brain project but hopefully can be used
|
156
185
|
in any other ruby project that needs to use cloud service providers' resources.
|
157
|
-
Version 0.2.
|
158
|
-
features you can find in the docs.
|
159
|
-
|
186
|
+
Version 0.2.6 has a little EC2, S3, SQS, SNS, Kinesis, websockets and a few other
|
187
|
+
features you can find in the docs. There is also limitted support for stubbing
|
188
|
+
AWS RESTful API calls. That can come in handy for local testing and extra setup on
|
189
|
+
AWS resource clients.
|
190
|
+
The next versions will have more Kinesis, Workflow integration and IoT.
|
160
191
|
This project is actively being developed, so more additions, specs and docs
|
161
192
|
will be added and updated frequently with new funcionality but the gem will
|
162
|
-
follow good practices for versioning.
|
193
|
+
follow good practices for versioning. Input is always welcome.
|
163
194
|
Enjoy!
|
164
|
-
email:
|
195
|
+
email:
|
196
|
+
- adam.phillipps@gmail.com
|
197
|
+
- Faisal@smashanalytics.com
|
165
198
|
executables: []
|
166
199
|
extensions: []
|
167
200
|
extra_rdoc_files: []
|
@@ -173,6 +206,7 @@ files:
|
|
173
206
|
- ".travis.yml"
|
174
207
|
- Gemfile
|
175
208
|
- Gemfile.lock
|
209
|
+
- Gemfile.lock_b
|
176
210
|
- LICENSE.txt
|
177
211
|
- README.md
|
178
212
|
- Rakefile
|
@@ -194,6 +228,8 @@ files:
|
|
194
228
|
- lib/cloud_powers/synapse/queue/board.rb
|
195
229
|
- lib/cloud_powers/synapse/queue/queue.rb
|
196
230
|
- lib/cloud_powers/synapse/synapse.rb
|
231
|
+
- lib/cloud_powers/synapse/websocket/websocclient.rb
|
232
|
+
- lib/cloud_powers/synapse/websocket/websocserver.rb
|
197
233
|
- lib/cloud_powers/version.rb
|
198
234
|
- lib/cloud_powers/workflow.rb
|
199
235
|
- lib/cloud_powers/zenv.rb
|