cloud_powers 0.2.7.23 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.test.env.example +6 -6
- data/.travis.yml +1 -1
- data/README +190 -0
- data/cloud_powers.gemspec +4 -4
- data/lib/cloud_powers.rb +3 -13
- data/lib/cloud_powers/aws_resources.rb +21 -4
- data/lib/cloud_powers/creatable.rb +122 -0
- data/lib/cloud_powers/helpers.rb +58 -0
- data/lib/cloud_powers/helpers/lang_help.rb +288 -0
- data/lib/cloud_powers/helpers/logic_help.rb +152 -0
- data/lib/cloud_powers/helpers/path_help.rb +90 -0
- data/lib/cloud_powers/node.rb +69 -68
- data/lib/cloud_powers/node/instance.rb +52 -0
- data/lib/cloud_powers/resource.rb +44 -0
- data/lib/cloud_powers/storage.rb +27 -14
- data/lib/{stubs → cloud_powers/stubs}/aws_stubs.rb +37 -14
- data/lib/cloud_powers/synapse/broadcast.rb +117 -0
- data/lib/cloud_powers/synapse/broadcast/channel.rb +44 -0
- data/lib/cloud_powers/synapse/pipe.rb +211 -0
- data/lib/cloud_powers/synapse/pipe/stream.rb +41 -0
- data/lib/cloud_powers/synapse/queue.rb +357 -0
- data/lib/cloud_powers/synapse/queue/board.rb +61 -95
- data/lib/cloud_powers/synapse/queue/poller.rb +29 -0
- data/lib/cloud_powers/synapse/synapse.rb +10 -12
- data/lib/cloud_powers/synapse/web_soc.rb +13 -0
- data/lib/cloud_powers/synapse/web_soc/soc_client.rb +52 -0
- data/lib/cloud_powers/synapse/web_soc/soc_server.rb +48 -0
- data/lib/cloud_powers/version.rb +1 -1
- data/lib/cloud_powers/zenv.rb +13 -12
- metadata +24 -13
- data/lib/cloud_powers/context.rb +0 -275
- data/lib/cloud_powers/delegator.rb +0 -113
- data/lib/cloud_powers/helper.rb +0 -453
- data/lib/cloud_powers/synapse/websocket/websocclient.rb +0 -53
- data/lib/cloud_powers/synapse/websocket/websocserver.rb +0 -46
- data/lib/cloud_powers/workflow_factory.rb +0 -160
@@ -1,47 +1,41 @@
|
|
1
|
+
require 'cloud_powers/aws_resources'
|
2
|
+
require 'cloud_powers/helpers'
|
3
|
+
require 'cloud_powers/resource'
|
4
|
+
require 'cloud_powers/synapse/queue'
|
5
|
+
require 'cloud_powers/zenv'
|
6
|
+
|
1
7
|
module Smash
|
2
8
|
module CloudPowers
|
3
9
|
module Synapse
|
4
10
|
module Queue
|
5
|
-
# The Queue::
|
11
|
+
# The Queue::Resource class helps wrap up information and functionality of a Queue on SQS.
|
6
12
|
# It is basically just an abstraction to make using SQS simpler
|
7
|
-
class Board
|
13
|
+
class Board < Smash::CloudPowers::Resource
|
8
14
|
include Smash::CloudPowers::AwsResources
|
9
15
|
include Smash::CloudPowers::Synapse::Queue
|
10
|
-
include Smash::CloudPowers::
|
16
|
+
include Smash::CloudPowers::Helpers
|
11
17
|
include Smash::CloudPowers::Zenv
|
12
18
|
|
13
19
|
# The URL the Aws::SQS::Queue uses
|
14
|
-
|
15
|
-
#
|
16
|
-
|
17
|
-
# An Aws::SQS::Client. See
|
20
|
+
attr_accessor :address
|
21
|
+
# +Hash+ response from Aws SDK <tt>Aws::SQS::Client#create_resource()</tt>
|
22
|
+
attr_reader :response
|
23
|
+
# An Aws::SQS::Client. See <tt>Smash::CloudPowers::AwsResources#sqs()<tt>
|
18
24
|
attr_accessor :sqs
|
19
25
|
|
20
|
-
# Creates a
|
26
|
+
# Creates a Resource object.
|
21
27
|
# The +#new()+ method is wrapped in +#build()+ and +#create!()+ but isn't private so
|
22
28
|
# +#new()+ can be used instead of build.
|
23
|
-
# The
|
29
|
+
# The Resource doesn't create Queue(s) or any other API calls until it is instructed to.
|
24
30
|
#
|
25
31
|
# Parameters
|
26
|
-
# * name +String+ - the name of the
|
32
|
+
# * name +String+ - the name of the resource can be used to find its arn/url etc
|
27
33
|
#
|
28
34
|
# Returns
|
29
|
-
# +Queue::
|
30
|
-
def initialize(name
|
31
|
-
|
32
|
-
@
|
33
|
-
end
|
34
|
-
|
35
|
-
# Gives back a string representation of the instance variable for this board.
|
36
|
-
#
|
37
|
-
# Returns +String+ - the instance variable for this Board in string format
|
38
|
-
#
|
39
|
-
# Example
|
40
|
-
# board = Board.new(:backlog)
|
41
|
-
# Smash.instance_variable_get(board.i_var)
|
42
|
-
# #=> Board <name: :backlog, ...>
|
43
|
-
def i_var
|
44
|
-
"@#{@name}"
|
35
|
+
# +Queue::Resource+
|
36
|
+
def initialize(name:, client: sqs, **config)
|
37
|
+
super
|
38
|
+
@sqs = client
|
45
39
|
end
|
46
40
|
|
47
41
|
# Gives the Queue address (URL). First the environment is searched, using Zenv and if nothing is
|
@@ -50,58 +44,17 @@ module Smash
|
|
50
44
|
# Returns
|
51
45
|
# * queue address <String>
|
52
46
|
def address
|
53
|
-
|
54
|
-
end
|
55
|
-
|
56
|
-
# Gives a best guess at the URL that points toward this Board's Queue. It uses a couple params
|
57
|
-
# to build a standard URL for SQS. The only problem with using this last resort is you may need
|
58
|
-
# to use a Queue from a different region, account or name but it can be a handy catch-all for the URLs
|
59
|
-
# for most cases.
|
60
|
-
#
|
61
|
-
# Returns String
|
62
|
-
# * exp. "https://sqs.us-west-2.amazonaws.com/12345678/fooBar"
|
63
|
-
def best_guess_address
|
64
|
-
"https://sqs.#{zfind(:aws_region)}.amazonaws.com/#{zfind(:account_number)}/#{@name}"
|
65
|
-
end
|
66
|
-
|
67
|
-
# Builds a Queue::Board object and returns it. No API calls are sent using this method. This is
|
68
|
-
# handy if you need to use a Queue but you don't want to create any resources in AWS.
|
69
|
-
#
|
70
|
-
# Parameters name +String+
|
71
|
-
#
|
72
|
-
# Returns Queue::Board
|
73
|
-
#
|
74
|
-
# Example
|
75
|
-
# exp_board = Board.build('exampleBoard')
|
76
|
-
# puts exp_board.best_guess_address
|
77
|
-
# # => 'https://sqs.us-west-2.amaz.....'
|
78
|
-
def self.build(name, this_sqs = nil)
|
79
|
-
new(name, this_sqs)
|
80
|
-
end
|
81
|
-
|
82
|
-
# Builds then Creates the Object and makes the API call to SQS to create the queue
|
83
|
-
#
|
84
|
-
# Parameters
|
85
|
-
# * name <String>
|
86
|
-
#
|
87
|
-
# Returns
|
88
|
-
# +Queue::Board+ with an actual Queue in SQS
|
89
|
-
#
|
90
|
-
# Example
|
91
|
-
# exp_board = Board.create!('exampleBoard')
|
92
|
-
# queue_search(exp_board.name)
|
93
|
-
# # => ['https://sqs.us-west-2.amazonaws.com/81234567890/exampleBoard']
|
94
|
-
def self.create!(name, this_sqs = nil)
|
95
|
-
build(name, this_sqs).create_queue!
|
47
|
+
@address ||= best_guess_address(name)
|
96
48
|
end
|
97
49
|
|
98
50
|
# Creates an actual Queue in SQS using the standard format for <b><i>this</i></b> queue name (camel case)
|
99
51
|
#
|
100
52
|
# Returns
|
101
|
-
# +Queue::
|
102
|
-
def
|
53
|
+
# +Queue::Resource+
|
54
|
+
def create_resource
|
103
55
|
begin
|
104
|
-
sqs.create_queue(queue_name: to_camel(@name))
|
56
|
+
@response = sqs.create_queue(queue_name: to_camel(@name))
|
57
|
+
yield self if block_given?
|
105
58
|
self
|
106
59
|
rescue Aws::SQS::Errors::QueueDeletedRecently
|
107
60
|
sleep 5
|
@@ -114,19 +67,48 @@ module Smash
|
|
114
67
|
sqs.delete_queue(queue_url: address)
|
115
68
|
end
|
116
69
|
|
70
|
+
# Gives back a string representation of the instance variable for this resource.
|
71
|
+
#
|
72
|
+
# Returns +String+ - the instance variable for this Resource in string format
|
73
|
+
#
|
74
|
+
# Example
|
75
|
+
# queue = Queue::Resource.new(:backlog)
|
76
|
+
# Smash.instance_variable_get(resource.i_var)
|
77
|
+
# #=> Resource <name: :backlog, ...>
|
78
|
+
def i_var
|
79
|
+
to_i_var(@name)
|
80
|
+
end
|
81
|
+
|
117
82
|
# Predicate method to query SQS for the queue
|
118
83
|
#
|
119
84
|
# Example
|
120
|
-
#
|
121
|
-
#
|
85
|
+
# queue = Queue::Resource.build('example')
|
86
|
+
# queue.exists?
|
122
87
|
# # => false
|
123
|
-
#
|
124
|
-
#
|
88
|
+
# queue.save!
|
89
|
+
# queue.exists?
|
125
90
|
# # => true
|
126
91
|
def exists?
|
127
92
|
queue_exists?(@name)
|
128
93
|
end
|
129
94
|
|
95
|
+
def link
|
96
|
+
if exists?
|
97
|
+
urls = queue_search(call_name)
|
98
|
+
|
99
|
+
if urls.size > 1
|
100
|
+
logger.info sitrep(content: "multiple matching #{name} queues to link to")
|
101
|
+
return @linked = false
|
102
|
+
end
|
103
|
+
|
104
|
+
# @url = urls.first
|
105
|
+
@url = sqs.get_queue_url(queue_name: @name).queue_url
|
106
|
+
else
|
107
|
+
save!
|
108
|
+
end
|
109
|
+
@linked = @url.eql? urls.first
|
110
|
+
end
|
111
|
+
|
130
112
|
# Gets the approximate message count for a Queue using the 'ApproximateMessageCount' attribute
|
131
113
|
#
|
132
114
|
# Returns
|
@@ -135,7 +117,7 @@ module Smash
|
|
135
117
|
get_queue_message_count(address)
|
136
118
|
end
|
137
119
|
|
138
|
-
# Gets a QueuePoller for the Queue attached to this
|
120
|
+
# Gets a QueuePoller for the Queue attached to this Resource instance.
|
139
121
|
#
|
140
122
|
# Returns
|
141
123
|
# +Aws::SQS::QueuePoller+
|
@@ -144,28 +126,12 @@ module Smash
|
|
144
126
|
# * Provide an existing SQS Client if one exists. This is used to sort out development
|
145
127
|
# production work.
|
146
128
|
def poller
|
147
|
-
|
148
|
-
@poller ||= Aws::SQS::QueuePoller.new(*args)
|
129
|
+
@poller ||= Aws::SQS::QueuePoller.new(queue_url: address, client: sqs)
|
149
130
|
end
|
150
131
|
|
151
132
|
# Retrieves a message from the Queue and deletes it from the Queue in SQS
|
152
133
|
def pluck_message
|
153
|
-
pluck_queue_message(
|
154
|
-
end
|
155
|
-
|
156
|
-
# This method creates the queue in SQS for the given Board instance
|
157
|
-
# It can be coupled with the #build() method in order to use a queue without
|
158
|
-
# making the call to create it on AWS
|
159
|
-
#
|
160
|
-
# Example
|
161
|
-
# board = Board.build('example')
|
162
|
-
# board.exists?
|
163
|
-
# # => false
|
164
|
-
# board.save!
|
165
|
-
# board.exists?
|
166
|
-
# # => true
|
167
|
-
def save!
|
168
|
-
create_queue!
|
134
|
+
pluck_queue_message(name)
|
169
135
|
end
|
170
136
|
|
171
137
|
# Sends the given message to the queue
|
@@ -0,0 +1,29 @@
|
|
1
|
+
require 'cloud_powers/aws_resources'
|
2
|
+
require 'cloud_powers/helpers'
|
3
|
+
require 'cloud_powers/resource'
|
4
|
+
require 'cloud_powers/synapse/queue'
|
5
|
+
require 'cloud_powers/zenv'
|
6
|
+
|
7
|
+
module Smash
|
8
|
+
module CloudPowers
|
9
|
+
module Synapse
|
10
|
+
module Queue
|
11
|
+
# The Queue::Resource class helps wrap up information and functionality of a Queue on SQS.
|
12
|
+
# It is basically just an abstraction to make using SQS simpler
|
13
|
+
class Poller < Smash::CloudPowers::Resource
|
14
|
+
attr_accessor :sqs
|
15
|
+
|
16
|
+
def initialize(name:, client: sqs, **config)
|
17
|
+
super
|
18
|
+
@sqs = client
|
19
|
+
@call_name = queue_poller_name(name)
|
20
|
+
end
|
21
|
+
|
22
|
+
def create_resource
|
23
|
+
@response = queue_poller(queue_url: address, client: sqs)
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
@@ -1,9 +1,7 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
require_relative './websocket/websocserver'
|
6
|
-
require_relative './websocket/websocclient'
|
1
|
+
require 'cloud_powers/synapse/broadcast'
|
2
|
+
require 'cloud_powers/synapse/queue'
|
3
|
+
require 'cloud_powers/synapse/pipe'
|
4
|
+
require 'cloud_powers/synapse/web_soc'
|
7
5
|
|
8
6
|
module Smash
|
9
7
|
module CloudPowers
|
@@ -14,13 +12,13 @@ module Smash
|
|
14
12
|
# Pipe is a module that is useful for sending large result sets, data to be processed
|
15
13
|
# or loaded, logging info and any other high-throughput/data-centric application with
|
16
14
|
include Smash::CloudPowers::Synapse::Pipe
|
17
|
-
#
|
18
|
-
# and any number of users or apps that
|
15
|
+
# MessageBoard is a module that is primarily used for asynchronous
|
16
|
+
# communications between a sender and any number of users or apps that
|
17
|
+
# _might_ need to use it. Messages on the board aren't ordered.
|
19
18
|
include Smash::CloudPowers::Synapse::Queue
|
20
|
-
#
|
21
|
-
|
22
|
-
|
23
|
-
include Smash::CloudPowers::Synapse::WebSocServer
|
19
|
+
# Socket is a module that allows Nodes to make a direct connection while
|
20
|
+
# still allowing others to read messages. Websockets are used
|
21
|
+
include Smash::CloudPowers::Synapse::WebSoc
|
24
22
|
end
|
25
23
|
end
|
26
24
|
end
|
@@ -0,0 +1,13 @@
|
|
1
|
+
require 'cloud_powers/synapse/web_soc/soc_client'
|
2
|
+
require 'cloud_powers/synapse/web_soc/soc_server'
|
3
|
+
|
4
|
+
module Smash
|
5
|
+
module CloudPowers
|
6
|
+
module Synapse
|
7
|
+
module WebSoc
|
8
|
+
include Smash::CloudPowers::Synapse::WebSoc::SocClient
|
9
|
+
include Smash::CloudPowers::Synapse::WebSoc::SocServer
|
10
|
+
end
|
11
|
+
end
|
12
|
+
end
|
13
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'websocket-eventmachine-client'
|
2
|
+
|
3
|
+
module Smash
|
4
|
+
module CloudPowers
|
5
|
+
module Synapse
|
6
|
+
module WebSoc
|
7
|
+
module SocClient
|
8
|
+
def create_websoc_client(opts = {})
|
9
|
+
ws = {}
|
10
|
+
Thread.new(ws) do
|
11
|
+
EM.run do
|
12
|
+
ws = WebSocket::EventMachine::Client.connect(:uri => 'ws://' + opts[:host] + ':' + opts[:port])
|
13
|
+
|
14
|
+
client_name = opts[:client] || 'default_client'
|
15
|
+
|
16
|
+
instance_variable_set(:"@#{client_name}",ws)
|
17
|
+
|
18
|
+
open_callback = opts[:on_open] || Proc.new do
|
19
|
+
puts "Connected"
|
20
|
+
end
|
21
|
+
|
22
|
+
ws.onopen &open_callback
|
23
|
+
|
24
|
+
on_message_callback = opts[:on_message] || Proc.new do |msg, type|
|
25
|
+
puts "Received message: #{msg}"
|
26
|
+
end
|
27
|
+
|
28
|
+
ws.onmessage &on_message_callback
|
29
|
+
|
30
|
+
on_error_callback = opts[:on_error] || Proc.new do |error|
|
31
|
+
puts "Error ==> #{error}"
|
32
|
+
end
|
33
|
+
|
34
|
+
ws.onerror &on_error_callback
|
35
|
+
|
36
|
+
on_close_callback = opts[:on_close] || Proc.new do |code, reason|
|
37
|
+
puts "Disconnected with status code: #{code}"
|
38
|
+
puts "Disconnected with status message: #{reason}"
|
39
|
+
end
|
40
|
+
|
41
|
+
ws.onclose &on_close_callback
|
42
|
+
|
43
|
+
end
|
44
|
+
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,48 @@
|
|
1
|
+
require 'websocket-eventmachine-server'
|
2
|
+
module Smash
|
3
|
+
module CloudPowers
|
4
|
+
module Synapse
|
5
|
+
module WebSoc
|
6
|
+
module SocServer
|
7
|
+
def create_websoc_server(opts = {})
|
8
|
+
channel = opts[:channel] || EM::Channel.new
|
9
|
+
Thread.new do
|
10
|
+
EM.run do
|
11
|
+
WebSocket::EventMachine::Server.start(:host => opts[:host], :port => opts[:port]) do |ws|
|
12
|
+
sid = nil
|
13
|
+
|
14
|
+
open_callback = opts[:on_open] || Proc.new do
|
15
|
+
puts "Client connected"
|
16
|
+
sid = channel.subscribe { |msg| ws.send msg }
|
17
|
+
end
|
18
|
+
ws.onopen &open_callback
|
19
|
+
|
20
|
+
on_message_callback = opts[:on_message] || Proc.new do |msg, type|
|
21
|
+
@current_websocket_message = msg
|
22
|
+
end
|
23
|
+
ws.onmessage &on_message_callback
|
24
|
+
|
25
|
+
on_error_callback = opts[:on_error] || Proc.new do |error|
|
26
|
+
puts "Error occured: #{error}"
|
27
|
+
end
|
28
|
+
ws.onerror &on_error_callback
|
29
|
+
|
30
|
+
on_close_callback = opts[:on_close] || Proc.new do
|
31
|
+
puts "Client disconnected"
|
32
|
+
channel.unsubscribe(sid) unless channel.nil?
|
33
|
+
end
|
34
|
+
ws.onclose &on_close_callback
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
channel
|
39
|
+
end
|
40
|
+
|
41
|
+
def broadcast_message(channel, msg)
|
42
|
+
channel.push msg
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
data/lib/cloud_powers/version.rb
CHANGED
data/lib/cloud_powers/zenv.rb
CHANGED
@@ -1,5 +1,5 @@
|
|
1
1
|
require 'dotenv'
|
2
|
-
|
2
|
+
require 'cloud_powers/helpers'
|
3
3
|
|
4
4
|
module Smash
|
5
5
|
module CloudPowers
|
@@ -9,7 +9,7 @@ module Smash
|
|
9
9
|
# System ENV, dotenv ENV and instance variables are considered for now but this
|
10
10
|
# will also use elasticache/redis...some other stuff too, in the coming versions
|
11
11
|
module Zenv
|
12
|
-
include Smash::CloudPowers::
|
12
|
+
include Smash::CloudPowers::Helpers
|
13
13
|
|
14
14
|
# Attempts to find a file by searching the current directory for the file
|
15
15
|
# then walking up the file tree and searching at each stop all the way up
|
@@ -51,12 +51,16 @@ module Smash
|
|
51
51
|
# Returns
|
52
52
|
# the value of the +key+ searched for
|
53
53
|
def i_vars(key = '')
|
54
|
+
name = to_i_var(key)
|
55
|
+
|
56
|
+
# if no key is given, return a +Hash+ of all i-var/value pairs
|
54
57
|
if key.empty?
|
55
|
-
return self.instance_variables.inject({}) do |r,v|
|
56
|
-
r.tap { |h| h[
|
58
|
+
return self.instance_variables.inject({}) do |r, v|
|
59
|
+
r.tap { |h| h[name] = self.instance_variable_get(name) }
|
57
60
|
end
|
58
61
|
end
|
59
|
-
|
62
|
+
|
63
|
+
self.instance_variable_get(name)
|
60
64
|
end
|
61
65
|
|
62
66
|
# PROJECT_ROOT should be set as early as possible in this Node's initilize
|
@@ -119,6 +123,7 @@ module Smash
|
|
119
123
|
# with this structure +{ key => value, ... }+ is returned for all keys with a value.
|
120
124
|
# Keys with no value are ommitted from the result.
|
121
125
|
def system_vars(key = '')
|
126
|
+
name = to_snake(key).upcase
|
122
127
|
if key.empty?
|
123
128
|
# Separate key-value pairs from the large string received by `ENV`
|
124
129
|
separate_pairs = `ENV`.split(/\n/).map do |string_pair|
|
@@ -131,14 +136,13 @@ module Smash
|
|
131
136
|
res.tap { |h_res| h_res[pair.first] = pair.last unless (pair.first == pair.last) }
|
132
137
|
end
|
133
138
|
else
|
134
|
-
|
135
|
-
return res.empty? ? nil : res
|
139
|
+
Object::ENV.has_key?(name) ? Object::ENV.fetch(name) : nil
|
136
140
|
end
|
137
141
|
end
|
138
142
|
|
139
143
|
# ZFind looks for the key in a preditermined order of importance:
|
140
144
|
# * i-vars are considered first becuase they might be tracking different
|
141
|
-
# locations for multiple
|
145
|
+
# locations for multiple jobs or something like that.
|
142
146
|
# * dotenv files are second because they were manually set, so for sure
|
143
147
|
# it's important
|
144
148
|
# * System Env[@] variables are up next. Hopefully by this time we've found
|
@@ -154,10 +158,7 @@ module Smash
|
|
154
158
|
# * TODO: implement a search for all 3 that can find close matches
|
155
159
|
def zfind(key)
|
156
160
|
project_root if @project_root.nil?
|
157
|
-
|
158
|
-
env_vars[to_snake(key).upcase] unless @project_root.nil?) or
|
159
|
-
system_vars[to_snake(key).upcase]
|
160
|
-
(res.nil? or res.empty?) ? nil : res
|
161
|
+
i_vars(key) || env_vars(key) || system_vars(key)
|
161
162
|
end
|
162
163
|
end
|
163
164
|
end
|