cloud_powers 0.2.7.23 → 1.0.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.test.env.example +6 -6
- data/.travis.yml +1 -1
- data/README +190 -0
- data/cloud_powers.gemspec +4 -4
- data/lib/cloud_powers.rb +3 -13
- data/lib/cloud_powers/aws_resources.rb +21 -4
- data/lib/cloud_powers/creatable.rb +122 -0
- data/lib/cloud_powers/helpers.rb +58 -0
- data/lib/cloud_powers/helpers/lang_help.rb +288 -0
- data/lib/cloud_powers/helpers/logic_help.rb +152 -0
- data/lib/cloud_powers/helpers/path_help.rb +90 -0
- data/lib/cloud_powers/node.rb +69 -68
- data/lib/cloud_powers/node/instance.rb +52 -0
- data/lib/cloud_powers/resource.rb +44 -0
- data/lib/cloud_powers/storage.rb +27 -14
- data/lib/{stubs → cloud_powers/stubs}/aws_stubs.rb +37 -14
- data/lib/cloud_powers/synapse/broadcast.rb +117 -0
- data/lib/cloud_powers/synapse/broadcast/channel.rb +44 -0
- data/lib/cloud_powers/synapse/pipe.rb +211 -0
- data/lib/cloud_powers/synapse/pipe/stream.rb +41 -0
- data/lib/cloud_powers/synapse/queue.rb +357 -0
- data/lib/cloud_powers/synapse/queue/board.rb +61 -95
- data/lib/cloud_powers/synapse/queue/poller.rb +29 -0
- data/lib/cloud_powers/synapse/synapse.rb +10 -12
- data/lib/cloud_powers/synapse/web_soc.rb +13 -0
- data/lib/cloud_powers/synapse/web_soc/soc_client.rb +52 -0
- data/lib/cloud_powers/synapse/web_soc/soc_server.rb +48 -0
- data/lib/cloud_powers/version.rb +1 -1
- data/lib/cloud_powers/zenv.rb +13 -12
- metadata +24 -13
- data/lib/cloud_powers/context.rb +0 -275
- data/lib/cloud_powers/delegator.rb +0 -113
- data/lib/cloud_powers/helper.rb +0 -453
- data/lib/cloud_powers/synapse/websocket/websocclient.rb +0 -53
- data/lib/cloud_powers/synapse/websocket/websocserver.rb +0 -46
- data/lib/cloud_powers/workflow_factory.rb +0 -160
@@ -1,47 +1,41 @@
|
|
1
|
+
require 'cloud_powers/aws_resources'
|
2
|
+
require 'cloud_powers/helpers'
|
3
|
+
require 'cloud_powers/resource'
|
4
|
+
require 'cloud_powers/synapse/queue'
|
5
|
+
require 'cloud_powers/zenv'
|
6
|
+
|
1
7
|
module Smash
|
2
8
|
module CloudPowers
|
3
9
|
module Synapse
|
4
10
|
module Queue
|
5
|
-
# The Queue::
|
11
|
+
# The Queue::Resource class helps wrap up information and functionality of a Queue on SQS.
|
6
12
|
# It is basically just an abstraction to make using SQS simpler
|
7
|
-
class Board
|
13
|
+
class Board < Smash::CloudPowers::Resource
|
8
14
|
include Smash::CloudPowers::AwsResources
|
9
15
|
include Smash::CloudPowers::Synapse::Queue
|
10
|
-
include Smash::CloudPowers::
|
16
|
+
include Smash::CloudPowers::Helpers
|
11
17
|
include Smash::CloudPowers::Zenv
|
12
18
|
|
13
19
|
# The URL the Aws::SQS::Queue uses
|
14
|
-
|
15
|
-
#
|
16
|
-
|
17
|
-
# An Aws::SQS::Client. See
|
20
|
+
attr_accessor :address
|
21
|
+
# +Hash+ response from Aws SDK <tt>Aws::SQS::Client#create_resource()</tt>
|
22
|
+
attr_reader :response
|
23
|
+
# An Aws::SQS::Client. See <tt>Smash::CloudPowers::AwsResources#sqs()<tt>
|
18
24
|
attr_accessor :sqs
|
19
25
|
|
20
|
-
# Creates a
|
26
|
+
# Creates a Resource object.
|
21
27
|
# The +#new()+ method is wrapped in +#build()+ and +#create!()+ but isn't private so
|
22
28
|
# +#new()+ can be used instead of build.
|
23
|
-
# The
|
29
|
+
# The Resource doesn't create Queue(s) or any other API calls until it is instructed to.
|
24
30
|
#
|
25
31
|
# Parameters
|
26
|
-
# * name +String+ - the name of the
|
32
|
+
# * name +String+ - the name of the resource can be used to find its arn/url etc
|
27
33
|
#
|
28
34
|
# Returns
|
29
|
-
# +Queue::
|
30
|
-
def initialize(name
|
31
|
-
|
32
|
-
@
|
33
|
-
end
|
34
|
-
|
35
|
-
# Gives back a string representation of the instance variable for this board.
|
36
|
-
#
|
37
|
-
# Returns +String+ - the instance variable for this Board in string format
|
38
|
-
#
|
39
|
-
# Example
|
40
|
-
# board = Board.new(:backlog)
|
41
|
-
# Smash.instance_variable_get(board.i_var)
|
42
|
-
# #=> Board <name: :backlog, ...>
|
43
|
-
def i_var
|
44
|
-
"@#{@name}"
|
35
|
+
# +Queue::Resource+
|
36
|
+
def initialize(name:, client: sqs, **config)
|
37
|
+
super
|
38
|
+
@sqs = client
|
45
39
|
end
|
46
40
|
|
47
41
|
# Gives the Queue address (URL). First the environment is searched, using Zenv and if nothing is
|
@@ -50,58 +44,17 @@ module Smash
|
|
50
44
|
# Returns
|
51
45
|
# * queue address <String>
|
52
46
|
def address
|
53
|
-
|
54
|
-
end
|
55
|
-
|
56
|
-
# Gives a best guess at the URL that points toward this Board's Queue. It uses a couple params
|
57
|
-
# to build a standard URL for SQS. The only problem with using this last resort is you may need
|
58
|
-
# to use a Queue from a different region, account or name but it can be a handy catch-all for the URLs
|
59
|
-
# for most cases.
|
60
|
-
#
|
61
|
-
# Returns String
|
62
|
-
# * exp. "https://sqs.us-west-2.amazonaws.com/12345678/fooBar"
|
63
|
-
def best_guess_address
|
64
|
-
"https://sqs.#{zfind(:aws_region)}.amazonaws.com/#{zfind(:account_number)}/#{@name}"
|
65
|
-
end
|
66
|
-
|
67
|
-
# Builds a Queue::Board object and returns it. No API calls are sent using this method. This is
|
68
|
-
# handy if you need to use a Queue but you don't want to create any resources in AWS.
|
69
|
-
#
|
70
|
-
# Parameters name +String+
|
71
|
-
#
|
72
|
-
# Returns Queue::Board
|
73
|
-
#
|
74
|
-
# Example
|
75
|
-
# exp_board = Board.build('exampleBoard')
|
76
|
-
# puts exp_board.best_guess_address
|
77
|
-
# # => 'https://sqs.us-west-2.amaz.....'
|
78
|
-
def self.build(name, this_sqs = nil)
|
79
|
-
new(name, this_sqs)
|
80
|
-
end
|
81
|
-
|
82
|
-
# Builds then Creates the Object and makes the API call to SQS to create the queue
|
83
|
-
#
|
84
|
-
# Parameters
|
85
|
-
# * name <String>
|
86
|
-
#
|
87
|
-
# Returns
|
88
|
-
# +Queue::Board+ with an actual Queue in SQS
|
89
|
-
#
|
90
|
-
# Example
|
91
|
-
# exp_board = Board.create!('exampleBoard')
|
92
|
-
# queue_search(exp_board.name)
|
93
|
-
# # => ['https://sqs.us-west-2.amazonaws.com/81234567890/exampleBoard']
|
94
|
-
def self.create!(name, this_sqs = nil)
|
95
|
-
build(name, this_sqs).create_queue!
|
47
|
+
@address ||= best_guess_address(name)
|
96
48
|
end
|
97
49
|
|
98
50
|
# Creates an actual Queue in SQS using the standard format for <b><i>this</i></b> queue name (camel case)
|
99
51
|
#
|
100
52
|
# Returns
|
101
|
-
# +Queue::
|
102
|
-
def
|
53
|
+
# +Queue::Resource+
|
54
|
+
def create_resource
|
103
55
|
begin
|
104
|
-
sqs.create_queue(queue_name: to_camel(@name))
|
56
|
+
@response = sqs.create_queue(queue_name: to_camel(@name))
|
57
|
+
yield self if block_given?
|
105
58
|
self
|
106
59
|
rescue Aws::SQS::Errors::QueueDeletedRecently
|
107
60
|
sleep 5
|
@@ -114,19 +67,48 @@ module Smash
|
|
114
67
|
sqs.delete_queue(queue_url: address)
|
115
68
|
end
|
116
69
|
|
70
|
+
# Gives back a string representation of the instance variable for this resource.
|
71
|
+
#
|
72
|
+
# Returns +String+ - the instance variable for this Resource in string format
|
73
|
+
#
|
74
|
+
# Example
|
75
|
+
# queue = Queue::Resource.new(:backlog)
|
76
|
+
# Smash.instance_variable_get(resource.i_var)
|
77
|
+
# #=> Resource <name: :backlog, ...>
|
78
|
+
def i_var
|
79
|
+
to_i_var(@name)
|
80
|
+
end
|
81
|
+
|
117
82
|
# Predicate method to query SQS for the queue
|
118
83
|
#
|
119
84
|
# Example
|
120
|
-
#
|
121
|
-
#
|
85
|
+
# queue = Queue::Resource.build('example')
|
86
|
+
# queue.exists?
|
122
87
|
# # => false
|
123
|
-
#
|
124
|
-
#
|
88
|
+
# queue.save!
|
89
|
+
# queue.exists?
|
125
90
|
# # => true
|
126
91
|
def exists?
|
127
92
|
queue_exists?(@name)
|
128
93
|
end
|
129
94
|
|
95
|
+
def link
|
96
|
+
if exists?
|
97
|
+
urls = queue_search(call_name)
|
98
|
+
|
99
|
+
if urls.size > 1
|
100
|
+
logger.info sitrep(content: "multiple matching #{name} queues to link to")
|
101
|
+
return @linked = false
|
102
|
+
end
|
103
|
+
|
104
|
+
# @url = urls.first
|
105
|
+
@url = sqs.get_queue_url(queue_name: @name).queue_url
|
106
|
+
else
|
107
|
+
save!
|
108
|
+
end
|
109
|
+
@linked = @url.eql? urls.first
|
110
|
+
end
|
111
|
+
|
130
112
|
# Gets the approximate message count for a Queue using the 'ApproximateMessageCount' attribute
|
131
113
|
#
|
132
114
|
# Returns
|
@@ -135,7 +117,7 @@ module Smash
|
|
135
117
|
get_queue_message_count(address)
|
136
118
|
end
|
137
119
|
|
138
|
-
# Gets a QueuePoller for the Queue attached to this
|
120
|
+
# Gets a QueuePoller for the Queue attached to this Resource instance.
|
139
121
|
#
|
140
122
|
# Returns
|
141
123
|
# +Aws::SQS::QueuePoller+
|
@@ -144,28 +126,12 @@ module Smash
|
|
144
126
|
# * Provide an existing SQS Client if one exists. This is used to sort out development
|
145
127
|
# production work.
|
146
128
|
def poller
|
147
|
-
|
148
|
-
@poller ||= Aws::SQS::QueuePoller.new(*args)
|
129
|
+
@poller ||= Aws::SQS::QueuePoller.new(queue_url: address, client: sqs)
|
149
130
|
end
|
150
131
|
|
151
132
|
# Retrieves a message from the Queue and deletes it from the Queue in SQS
|
152
133
|
def pluck_message
|
153
|
-
pluck_queue_message(
|
154
|
-
end
|
155
|
-
|
156
|
-
# This method creates the queue in SQS for the given Board instance
|
157
|
-
# It can be coupled with the #build() method in order to use a queue without
|
158
|
-
# making the call to create it on AWS
|
159
|
-
#
|
160
|
-
# Example
|
161
|
-
# board = Board.build('example')
|
162
|
-
# board.exists?
|
163
|
-
# # => false
|
164
|
-
# board.save!
|
165
|
-
# board.exists?
|
166
|
-
# # => true
|
167
|
-
def save!
|
168
|
-
create_queue!
|
134
|
+
pluck_queue_message(name)
|
169
135
|
end
|
170
136
|
|
171
137
|
# Sends the given message to the queue
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'cloud_powers/aws_resources'
|
2
|
+
require 'cloud_powers/helpers'
|
3
|
+
require 'cloud_powers/resource'
|
4
|
+
require 'cloud_powers/synapse/queue'
|
5
|
+
require 'cloud_powers/zenv'
|
6
|
+
|
7
|
+
module Smash
|
8
|
+
module CloudPowers
|
9
|
+
module Synapse
|
10
|
+
module Queue
|
11
|
+
# The Queue::Resource class helps wrap up information and functionality of a Queue on SQS.
|
12
|
+
# It is basically just an abstraction to make using SQS simpler
|
13
|
+
class Poller < Smash::CloudPowers::Resource
|
14
|
+
attr_accessor :sqs
|
15
|
+
|
16
|
+
def initialize(name:, client: sqs, **config)
|
17
|
+
super
|
18
|
+
@sqs = client
|
19
|
+
@call_name = queue_poller_name(name)
|
20
|
+
end
|
21
|
+
|
22
|
+
def create_resource
|
23
|
+
@response = queue_poller(queue_url: address, client: sqs)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -1,9 +1,7 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
require_relative './websocket/websocserver'
|
6
|
-
require_relative './websocket/websocclient'
|
1
|
+
require 'cloud_powers/synapse/broadcast'
|
2
|
+
require 'cloud_powers/synapse/queue'
|
3
|
+
require 'cloud_powers/synapse/pipe'
|
4
|
+
require 'cloud_powers/synapse/web_soc'
|
7
5
|
|
8
6
|
module Smash
|
9
7
|
module CloudPowers
|
@@ -14,13 +12,13 @@ module Smash
|
|
14
12
|
# Pipe is a module that is useful for sending large result sets, data to be processed
|
15
13
|
# or loaded, logging info and any other high-throughput/data-centric application with
|
16
14
|
include Smash::CloudPowers::Synapse::Pipe
|
17
|
-
#
|
18
|
-
# and any number of users or apps that
|
15
|
+
# MessageBoard is a module that is primarily used for asynchronous
|
16
|
+
# communications between a sender and any number of users or apps that
|
17
|
+
# _might_ need to use it. Messages on the board aren't ordered.
|
19
18
|
include Smash::CloudPowers::Synapse::Queue
|
20
|
-
#
|
21
|
-
|
22
|
-
|
23
|
-
include Smash::CloudPowers::Synapse::WebSocServer
|
19
|
+
# Socket is a module that allows Nodes to make a direct connection while
|
20
|
+
# still allowing others to read messages. Websockets are used
|
21
|
+
include Smash::CloudPowers::Synapse::WebSoc
|
24
22
|
end
|
25
23
|
end
|
26
24
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'cloud_powers/synapse/web_soc/soc_client'
|
2
|
+
require 'cloud_powers/synapse/web_soc/soc_server'
|
3
|
+
|
4
|
+
module Smash
|
5
|
+
module CloudPowers
|
6
|
+
module Synapse
|
7
|
+
module WebSoc
|
8
|
+
include Smash::CloudPowers::Synapse::WebSoc::SocClient
|
9
|
+
include Smash::CloudPowers::Synapse::WebSoc::SocServer
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'websocket-eventmachine-client'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module Synapse
|
6
|
+
module WebSoc
|
7
|
+
module SocClient
|
8
|
+
def create_websoc_client(opts = {})
|
9
|
+
ws = {}
|
10
|
+
Thread.new(ws) do
|
11
|
+
EM.run do
|
12
|
+
ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://' + opts[:host] + ':' + opts[:port])
|
13
|
+
|
14
|
+
client_name = opts[:client] || 'default_client'
|
15
|
+
|
16
|
+
instance_variable_set(:"@#{client_name}",ws)
|
17
|
+
|
18
|
+
open_callback = opts[:on_open] || Proc.new do
|
19
|
+
puts "Connected"
|
20
|
+
end
|
21
|
+
|
22
|
+
ws.onopen &open_callback
|
23
|
+
|
24
|
+
on_message_callback = opts[:on_message] || Proc.new do |msg, type|
|
25
|
+
puts "Received message: #{msg}"
|
26
|
+
end
|
27
|
+
|
28
|
+
ws.onmessage &on_message_callback
|
29
|
+
|
30
|
+
on_error_callback = opts[:on_error] || Proc.new do |error|
|
31
|
+
puts "Error ==> #{error}"
|
32
|
+
end
|
33
|
+
|
34
|
+
ws.onerror &on_error_callback
|
35
|
+
|
36
|
+
on_close_callback = opts[:on_close] || Proc.new do |code, reason|
|
37
|
+
puts "Disconnected with status code: #{code}"
|
38
|
+
puts "Disconnected with status message: #{reason}"
|
39
|
+
end
|
40
|
+
|
41
|
+
ws.onclose &on_close_callback
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'websocket-eventmachine-server'
|
2
|
+
module Smash
|
3
|
+
module CloudPowers
|
4
|
+
module Synapse
|
5
|
+
module WebSoc
|
6
|
+
module SocServer
|
7
|
+
def create_websoc_server(opts = {})
|
8
|
+
channel = opts[:channel] || EM::Channel.new
|
9
|
+
Thread.new do
|
10
|
+
EM.run do
|
11
|
+
WebSocket::EventMachine::Server.start(:host => opts[:host], :port => opts[:port]) do |ws|
|
12
|
+
sid = nil
|
13
|
+
|
14
|
+
open_callback = opts[:on_open] || Proc.new do
|
15
|
+
puts "Client connected"
|
16
|
+
sid = channel.subscribe { |msg| ws.send msg }
|
17
|
+
end
|
18
|
+
ws.onopen &open_callback
|
19
|
+
|
20
|
+
on_message_callback = opts[:on_message] || Proc.new do |msg, type|
|
21
|
+
@current_websocket_message = msg
|
22
|
+
end
|
23
|
+
ws.onmessage &on_message_callback
|
24
|
+
|
25
|
+
on_error_callback = opts[:on_error] || Proc.new do |error|
|
26
|
+
puts "Error occured: #{error}"
|
27
|
+
end
|
28
|
+
ws.onerror &on_error_callback
|
29
|
+
|
30
|
+
on_close_callback = opts[:on_close] || Proc.new do
|
31
|
+
puts "Client disconnected"
|
32
|
+
channel.unsubscribe(sid) unless channel.nil?
|
33
|
+
end
|
34
|
+
ws.onclose &on_close_callback
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
channel
|
39
|
+
end
|
40
|
+
|
41
|
+
def broadcast_message(channel, msg)
|
42
|
+
channel.push msg
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/cloud_powers/version.rb
CHANGED
data/lib/cloud_powers/zenv.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'dotenv'
|
2
|
-
|
2
|
+
require 'cloud_powers/helpers'
|
3
3
|
|
4
4
|
module Smash
|
5
5
|
module CloudPowers
|
@@ -9,7 +9,7 @@ module Smash
|
|
9
9
|
# System ENV, dotenv ENV and instance variables are considered for now but this
|
10
10
|
# will also use elasticache/redis...some other stuff too, in the coming versions
|
11
11
|
module Zenv
|
12
|
-
include Smash::CloudPowers::
|
12
|
+
include Smash::CloudPowers::Helpers
|
13
13
|
|
14
14
|
# Attempts to find a file by searching the current directory for the file
|
15
15
|
# then walking up the file tree and searching at each stop all the way up
|
@@ -51,12 +51,16 @@ module Smash
|
|
51
51
|
# Returns
|
52
52
|
# the value of the +key+ searched for
|
53
53
|
def i_vars(key = '')
|
54
|
+
name = to_i_var(key)
|
55
|
+
|
56
|
+
# if no key is given, return a +Hash+ of all i-var/value pairs
|
54
57
|
if key.empty?
|
55
|
-
return self.instance_variables.inject({}) do |r,v|
|
56
|
-
r.tap { |h| h[
|
58
|
+
return self.instance_variables.inject({}) do |r, v|
|
59
|
+
r.tap { |h| h[name] = self.instance_variable_get(name) }
|
57
60
|
end
|
58
61
|
end
|
59
|
-
|
62
|
+
|
63
|
+
self.instance_variable_get(name)
|
60
64
|
end
|
61
65
|
|
62
66
|
# PROJECT_ROOT should be set as early as possible in this Node's initilize
|
@@ -119,6 +123,7 @@ module Smash
|
|
119
123
|
# with this structure +{ key => value, ... }+ is returned for all keys with a value.
|
120
124
|
# Keys with no value are ommitted from the result.
|
121
125
|
def system_vars(key = '')
|
126
|
+
name = to_snake(key).upcase
|
122
127
|
if key.empty?
|
123
128
|
# Separate key-value pairs from the large string received by `ENV`
|
124
129
|
separate_pairs = `ENV`.split(/\n/).map do |string_pair|
|
@@ -131,14 +136,13 @@ module Smash
|
|
131
136
|
res.tap { |h_res| h_res[pair.first] = pair.last unless (pair.first == pair.last) }
|
132
137
|
end
|
133
138
|
else
|
134
|
-
|
135
|
-
return res.empty? ? nil : res
|
139
|
+
Object::ENV.has_key?(name) ? Object::ENV.fetch(name) : nil
|
136
140
|
end
|
137
141
|
end
|
138
142
|
|
139
143
|
# ZFind looks for the key in a preditermined order of importance:
|
140
144
|
# * i-vars are considered first becuase they might be tracking different
|
141
|
-
# locations for multiple
|
145
|
+
# locations for multiple jobs or something like that.
|
142
146
|
# * dotenv files are second because they were manually set, so for sure
|
143
147
|
# it's important
|
144
148
|
# * System Env[@] variables are up next. Hopefully by this time we've found
|
@@ -154,10 +158,7 @@ module Smash
|
|
154
158
|
# * TODO: implement a search for all 3 that can find close matches
|
155
159
|
def zfind(key)
|
156
160
|
project_root if @project_root.nil?
|
157
|
-
|
158
|
-
env_vars[to_snake(key).upcase] unless @project_root.nil?) or
|
159
|
-
system_vars[to_snake(key).upcase]
|
160
|
-
(res.nil? or res.empty?) ? nil : res
|
161
|
+
i_vars(key) || env_vars(key) || system_vars(key)
|
161
162
|
end
|
162
163
|
end
|
163
164
|
end
|