cloud_powers 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 70615a0db0f98f47026a8d05b715553490dfeb00
-  data.tar.gz: 7e2aca01ee8b38955586ae0d3fb42029d5bc9457
+  metadata.gz: 345005638bc5e69081209792b8b9884ba2db2605
+  data.tar.gz: f5b69e0a4d366da8e6512b2ea6062c18bf17dc34
 SHA512:
-  metadata.gz: b1da98d073d2d3fe8af5200fecd185ef8af1cba65672aba687780dd3f1d0b5fe52110b5470467a0d8f01bc40d6fcbf2b4883780e6e60b0d7d69c6a2062edee86
-  data.tar.gz: 6850a5a5e8c354599cd6ed18d437ff28961140ebacbede6047ab9728f544f8cac0219119c31e511424491acaad8e63d3d8c818a2345cd1747091f4e0da0d75fc
+  metadata.gz: 09ec8ecacab90dc97ea58cfe3b70c1359d3ed381076403edbca261265bc9eeee71d5e1ace2b5dea55401eb57d2c79c2ea258aef8242846c75ccd57d1eceb2542
+  data.tar.gz: e4c21cdb59a03d37fd45d4dba3cf8e3b5d6732a3deaa6351ef7c24220035f5cc42c2f78a81fec804abed81f8d211b3d6441caeecc6d26d021fb93479a2a0d8de

data/.gitignore

@@ -9,3 +9,5 @@
 /tmp/
 *.env
 .byebug_history
+lib/cloud_powers/synapse/.DS_Store
+.DS_Store

data/.test.env.example

@@ -4,6 +4,7 @@ PROJECT_ROOT=""
 # Aws access:
 AWS_ACCESS_KEY_ID=""
 AWS_SECRET_ACCESS_KEY=""
+ACCOUNT_NUMBER=""
 
 # Aws areas and auth-related locations: 
 AWS_REGION=""

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cloud_powers (0.2.2)
+    cloud_powers (0.2.3)
       activesupport-core-ext (~> 4)
       aws-sdk (~> 2)
       dotenv (~> 2.1)
@@ -13,12 +13,12 @@ GEM
   remote: https://rubygems.org/
   specs:
     activesupport-core-ext (4.0.0.3)
-    aws-sdk (2.5.8)
-      aws-sdk-resources (= 2.5.8)
-    aws-sdk-core (2.5.8)
+    aws-sdk (2.6.3)
+      aws-sdk-resources (= 2.6.3)
+    aws-sdk-core (2.6.3)
       jmespath (~> 1.0)
-    aws-sdk-resources (2.5.8)
-      aws-sdk-core (= 2.5.8)
+    aws-sdk-resources (2.6.3)
+      aws-sdk-core (= 2.6.3)
     byebug (9.0.5)
     diff-lcs (1.2.5)
     dotenv (2.1.1)

data/cloud_powers.gemspec

@@ -14,9 +14,10 @@ Gem::Specification.new do |spec|
     CloudPowers is a wrapper around AWS and other cloud services.
     It was developed specifically for the Brain project
     but hopefully can be used in any other ruby project that needs to use cloud
-    service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS and
-    Kinesis.  This project is actively being developed, so more additions, specs
-    and docs will be updated frequently.  I always welcome input.
+    service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS, SNS
+    and Kinesis.  V-0.2.4 should be out in a day or so and it will include IoT
+    basics and specs too.  This project is actively being developed, so more
+    additions, specs and docs will be updated frequently.  I always welcome input.
     Enjoy!
   EOF
   spec.homepage               =       'https://github.com/adam-phillipps/cloud_powers'

data/lib/cloud_powers/auth.rb

@@ -1,13 +1,26 @@
 require 'aws-sdk'
 Aws.use_bundled_cert!
+require_relative 'zenv'
 
 module Smash
   module CloudPowers
     module Auth
+      extend Smash::CloudPowers::Zenv
+
+      # This method is able to be called before an object is instantiated in order
+      # to provide a region in AWS-land.
+      def self.region
+        zfind(:aws_region) || 'us-west-2'
+      end
+
+      # This method is able to be called before an object is instantiated in order
+      # to provide an Aws::Credentials object that will allow access to all the
+      # resources in the account that zfind searches for, using the "ACCOUNT_NUMBER"
+      # key.
       def self.creds
         @creds ||= Aws::Credentials.new(
-          ENV['AWS_ACCESS_KEY_ID'],
-          ENV['AWS_SECRET_ACCESS_KEY']
+          zfind(:aws_access_key_id),
+          zfind(:aws_secret_access_key)
         )
       end
     end

data/lib/cloud_powers/aws_resources.rb

@@ -10,7 +10,7 @@ module Smash
       include Smash::CloudPowers::Zenv
 
       def region
-        zfind('Aws Region')
+        zfind(:aws_region) || 'us-west-2'
       end
 
       def ec2
@@ -34,15 +34,23 @@ module Smash
       end
 
       def s3
-        byebug
         @s3 ||= Aws::S3::Client.new(
           region: region,
           credentials: Auth.creds
         )
       end
 
+      def sns
+        @sns ||= Aws::SNS::Client.new(
+          region: region,
+          credentials: Auth.creds
+        )
+      end
+
       def sqs
-        @sqs ||= Aws::SQS::Client.new(credentials: Auth.creds)
+        @sqs ||= Aws::SQS::Client.new(
+          credentials: Auth.creds
+        )
       end
     end
   end

data/lib/cloud_powers/helper.rb

@@ -2,13 +2,11 @@ require 'logger'
 require 'pathname'
 require 'syslog/logger'
 require_relative 'smash_error'
-require_relative 'auth'
 require 'byebug'
 
 module Smash
   module CloudPowers
     module Helper
-      include Smash::CloudPowers::Auth
 
       def attr_map!(keys)
         keys.map do |attr|

data/lib/cloud_powers/self_awareness.rb

@@ -139,8 +139,11 @@ module Smash
         end
       end
 
+      # This method will return true if:
+      # * The run time is more than 5 minutes
+      # and
+      # * The run time is 5 minutes from the hour mark from when the instance started
       def time_is_up?
-        # returns true when the hour mark approaches
         an_hours_time = 60 * 60
         five_minutes_time = 60 * 5
 

data/lib/cloud_powers/synapse/broadcast/broadcast.rb

@@ -0,0 +1,85 @@
+module Smash
+  module CloudPowers
+    module Synapse
+      module Broadcast
+        include Smash::CloudPowers::Helper
+        include Smash::CloudPowers::AwsResources
+        include Smash::CloudPowers::Zenv
+
+        # A simple Struct to bind the name with the arn of the topic
+        Channel = Struct.new(:set_name, :set_arn, :set_endpoint) do
+          include Smash::CloudPowers::Zenv
+
+          # Prefers the given arn but it can make a best guess if none is given
+          def arn
+            set_arn || "arn:aws:sns:#{zfind(:region)}:#{zfind(:accound_number)}:#{set_name}"
+          end
+
+          # Prefers the given name but it can parse the arn to find one
+          def name
+            set_name || set_arn.split(':').last
+          end
+        end # end Channel
+        #################
+
+        # Creates a connection point for 1..N nodes to create a connection with the Broadcast
+        # def create_distributor(channel)
+        #   sns.create_application_platform()
+        # end
+
+        # Creates a point to connect to for information about a given topic
+        # @params: name <String>: the name of the Channel/Topic to be created
+        # @returns: Broadcast::Channel representing the created channel
+        def create_channel!(name)
+          resp = sns.create_topic(name: name)
+          Channel.new(nil, resp.topic_arn)
+        end
+
+        # Deletes a topic from SNS-land
+        # @params: channel <Broadcast::Channel>
+        def delete_channel!(channel)
+          sns.delete_topic(topic_arn: channel.arn)
+        end
+
+        # Creates a connection to the Broadcast so that new messages will be picked up
+        # @params: channel <Broadcast::Channel>
+        def listen_on(channel)
+          sns.subscribe(
+            topic_arn:    channel.arn,
+            protocol:     'application',
+            endpoint:     channel.endpoint
+          )
+        end
+
+        # Lists the created topics in SNS.
+        # @returns results <Array
+        def real_channels
+          results = []
+          next_token = ''
+          loop do
+            resp = sns.list_topics((next_token.empty? ? {} : { next_token: next_token }))
+            results.concat(resp.topics.map(&:topic_arn))
+            next_token = (resp.next_token.empty? ? '' : resp.next_token)
+            break if next_token.empty?
+          end
+          results
+        end
+
+        # Send a message to a Channel using SNS#publish
+        # @params: [opts <Hash>]:
+        #   this includes all the keys AWS uses but for now it only has defaults
+        #   for topic_arn and the message
+        def send_broadcast(opts = {})
+          msg = opts.delete(:message) || ""
+
+          package = {
+            topic_arn:            "topicARN",
+            message:              msg.to_json
+          }.merge(opts)
+
+          sns.publish(package)
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/synapse/{pipe.rb → pipe/pipe.rb}

@@ -1,4 +1,4 @@
-require_relative '../helper'
+require_relative '../../helper'
 
 module Smash
   module CloudPowers

data/lib/cloud_powers/synapse/queue/board.rb

@@ -0,0 +1,116 @@
+module Smash
+  module CloudPowers
+    module Synapse
+      module Queue
+        # The Queue::Board class helps wrap up information and functionality of a Queue on SQS.
+        # It is basically just an abstraction to make using SQS simpler
+        class Board
+          include Smash::CloudPowers::AwsResources
+          include Smash::CloudPowers::Synapse::Queue
+          include Smash::CloudPowers::Helper
+          include Smash::CloudPowers::Zenv
+
+          attr_accessor :address, :name, :poller
+
+          # Takes a `name` and creates a Board object.
+          # The #new method is wrapped in #build() and #create!() but isn't labeled private so
+          # #new can be used instead of build.
+          # The Board doesn't create Queue(s) or any other API calls until it is instructed to.
+          # @params: name <String>: the name of the board can be used to find its arn/url etc
+          # @returns: Queue::Board
+          # not.new
+          def initialize(name)
+            @name = name
+          end
+
+          # Gives back a string representation of the instance variable for this board.
+          # @returns: instance_variable <String>: the instance variable for this Board in string format
+          # Example:
+          #   board = Board.new(:backlog)
+          #   Smash.instance_variable_get(board.i_var)
+          #   #=> Board <name: :backlog ...>
+          def i_var
+            "@#{@name}"
+          end
+
+          # Gives the Queue address (URL).  First the environment is searched, using Zenv and if nothing is
+          # found, the best guess attempt at the correct address is used.
+          # @returns: queue address <String>
+          def address
+            zfind(@name) ||  best_guess_address
+          end
+
+          # Gives a best guess at the URL that points toward this Board's Queue.  It uses a couple params
+          # to build a standard URL for SQS.  The only problem with using this last resort is you may need
+          # to use a Queue from a different region, account or name but it can be a handy catch-all for the URLs
+          # for most cases.
+          def best_guess_address
+            "https://sqs.#{zfind(:aws_region)}.amazonaws.com/#{zfind(:account_number)}/#{@name}"
+          end
+
+          # Builds a Queue::Board object and returns it.  No API calls are sent using this method
+          # @params: name <String>
+          # @returns: Queue::Board
+          def self.build(name)
+            new(name)
+          end
+
+          # Builds then Creates the Object and makes the API call to SQS to create the queue
+          # @params: name <String>
+          # @returns: Queue::Board with an actual queue in SQS
+          def self.create!(name)
+            built_board = build(name)
+            built_board.create_queue!
+          end
+
+          # Creates an actual Queue in SQS using the standard format for a queue name (camel case)
+          # @returns: Queue::Board
+          def create_queue!
+            sqs.create_queue(queue_name: to_camel(@name))
+            self
+          end
+
+          # Deletes an actual Queue from SQS
+          def destroy!
+            sqs.delete_queue(queue_url: address)
+          end
+
+          # Predicate method to query SQS for the queue
+          def exists?
+            queue_exists?(@name)
+          end
+
+          # Gets the approximate message count for a Queue using the 'ApproximateMessageCount' attribute
+          def message_count
+            get_queue_message_count(address)
+          end
+
+          # Gets a QueuePoller for the Queue attached to this Board instance
+          def poller
+            @poller ||= Aws::SQS::QueuePoller.new(address)
+          end
+
+          # Retrieves a message from the Queue and deletes it from the Queue in SQS
+          def pluck_message
+            pluck_queue_message(@name)
+          end
+
+          # This method creates the queue in SQS for the given Board instance
+          # It can be coupled with the #build() method in order to use a queue without
+          # making the call to create it on AWS
+          def save!
+            create_queue!
+          end
+
+          # Sends the given message to the queue
+          # @params: message, which is either used as JSON or converted into it
+          def send_message(message)
+            send_queue_message(
+              address, (valid_json?(message) ? message : message.to_json)
+            )
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/synapse/queue/queue.rb

@@ -0,0 +1,128 @@
+require 'uri'
+require_relative 'board'
+
+module Smash
+  module CloudPowers
+    module Synapse
+      module Queue
+        include Smash::CloudPowers::Helper
+        include Smash::CloudPowers::AwsResources
+
+        # This method can be used to parse a queue name from its address.  It can be handy if you need the name
+        # of a queue but you don't want the overhead of creating a Board object.
+        def board_name(url)
+          url.to_s.split('/').last
+        end
+
+        # This method allows you to create a queue on SQS without explicitly creating a Board object
+        # @params: name <String>: The name of the queue to be created
+        # @returns: Queue::Board
+        def create_queue!(name)
+          begin
+            Board.create!(to_camel(name))
+          rescue Aws::SQS::Errors::QueueDeletedRecently => e
+            sleep 5
+            retry
+          end
+        end
+
+        # This method builds a Queue::Board object for you to use but doesn't invoke the #create! method, so
+        # no API call is made to create the queue on SQS.  This can be used if the board already exists.
+        # @params: name <String>: name of the queue you want to interact with
+        # @returns: Queue::Board
+        def build_queue(name)
+          Board.build(to_camel(name))
+        end
+
+        # Deletes a queue message without caring about reading/interacting with the message.
+        # This is usually used for progress tracking, ie; a Neuron takes a message from the Backlog, moves it to
+        # WIP and deletes it from Backlog.  Then repeats these steps for the remaining States in the Workflow
+        # @params: queue <String>[, opts <Hash>]
+        #   queue is the name of the queue to interact with
+        #   opts is a configuration hash for the SQS::QueuePoller
+        def delete_queue_message(queue, opts = {})
+          poll(queue, opts) do |msg, stats|
+            poller(queue).delete_message(msg)
+            throw :stop_polling
+          end
+        end
+
+        # This method is used to gain the approximate count of messages in a given queue
+        # @params: board_url <String>: The URL for the board you need to get a count from
+        # @returns: float representation of the count
+        def get_queue_message_count(board_url)
+          sqs.get_queue_attributes(
+            queue_url: board_url,
+            attribute_names: ['ApproximateNumberOfMessages']
+          ).attributes['ApproximateNumberOfMessages'].to_f
+        end
+
+        # @params: board<String|symbol>: The name of the board
+        # @returns a message and deletes it from its origin
+        def pluck_queue_message(board)
+          poll(board) do |msg, poller|
+            poller.delete_message(msg)
+            return valid_json?(msg.body) ? JSON.parse(msg.body) : msg.body
+          end
+        end
+
+        # Polls the given board with the given options hash and a block that interacts with
+        # the message that is retrieved from the queue
+        # @params: board <String>[, opts <Hash>]
+        #   board is the name of the queue you want to poll
+        #   opts can have any AWS::SQS polling option
+        #   &block is the block that is used to interact with the message that was retrieved
+        # @returns the results from the `message` and the `block` that interacts with the message(s)
+        def poll(board, opts = {})
+          this_poller = queue_poller(board)
+          results = nil
+          this_poller.poll(opts) do |msg|
+            results = yield msg, this_poller if block_given?
+            this_poller.delete_message(msg)
+            throw :stop_polling
+          end
+          results
+        end
+
+        # This method can be used to gain a SQS::QueuePoller.  It creates a Board object,
+        # the Board then sends the API call to SQS to create the queue and sets an instance
+        # variable, using the board's name, to the Board object itself
+        # @params: board_name <String>: name of the Queue you want to gain a QueuePoller for
+        # @returns: @<board_name:Queue::Board>
+        def queue_poller(board_name)
+          board = Board.create!(board_name)
+
+          unless instance_variable_defined?(board.i_var)
+            instance_variable_set(
+              board.i_var,
+              board
+            )
+          end
+          instance_variable_get(board.i_var).poller
+        end
+
+        # Checks SQS for the existence of this queue
+        def queue_exists?(name)
+          !sqs.list_queues(queue_name_prefix: name).queue_urls.empty?
+        end
+
+        # Searches for a queue based on the name
+        # @params: name <String>
+        # @returns: queue_urls <Array>
+        def queue_search(name)
+          sqs.list_queues(queue_name_prefix: name).queue_urls
+        end
+
+        # Sends a given message to a given queue
+        # @params: address <String>: address of the queue you want to interact with
+        # @returns: queue_urls <Array<String>>
+        def send_queue_message(address, message)
+          sqs.send_message(
+            queue_url: address,
+            message_body: message
+          )
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/synapse/synapse.rb

@@ -1,9 +1,18 @@
-require_relative 'queue'
-require_relative 'pipe'
+require_relative './broadcast/broadcast'
+require_relative './queue/board'
+require_relative './queue/queue'
+require_relative './pipe/pipe'
 
 module Smash
   module CloudPowers
+    # The Synapse module provides all communications functionality
+    # - Broadcast is a module that is useful for sending 1 message to multiple recipients
+    # - Pipe is a module that is useful for sending large result sets, data to be processed
+    #     or loaded, logging info and any other high-throughput/data-centric application with
+    # - Queue is a module that is primarily used for asynchronous communications between a sender
+    #     and any number of users or apps that _might_ need to use it
     module Synapse
+      include Smash::CloudPowers::Synapse::Broadcast
       include Smash::CloudPowers::Synapse::Pipe
       include Smash::CloudPowers::Synapse::Queue
     end

data/lib/cloud_powers/version.rb

@@ -1,3 +1,3 @@
 module CloudPowers
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloud_powers
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Adam Phillipps
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-27 00:00:00.000000000 Z
+date: 2016-09-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport-core-ext
@@ -154,9 +154,10 @@ description: |2
       CloudPowers is a wrapper around AWS and other cloud services.
       It was developed specifically for the Brain project
       but hopefully can be used in any other ruby project that needs to use cloud
-      service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS and
-      Kinesis.  This project is actively being developed, so more additions, specs
-      and docs will be updated frequently.  I always welcome input.
+      service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS, SNS
+      and Kinesis.  V-0.2.4 should be out in a day or so and it will include IoT
+      basics and specs too.  This project is actively being developed, so more
+      additions, specs and docs will be updated frequently.  I always welcome input.
       Enjoy!
 email: adam.phillipps@gmail.com
 executables: []
@@ -177,6 +178,7 @@ files:
 - bin/setup
 - cloud_powers.gemspec
 - lib/cloud_powers.rb
+- lib/cloud_powers/.DS_Store
 - lib/cloud_powers/auth.rb
 - lib/cloud_powers/aws_resources.rb
 - lib/cloud_powers/delegator.rb
@@ -184,8 +186,10 @@ files:
 - lib/cloud_powers/self_awareness.rb
 - lib/cloud_powers/smash_error.rb
 - lib/cloud_powers/storage.rb
-- lib/cloud_powers/synapse/pipe.rb
-- lib/cloud_powers/synapse/queue.rb
+- lib/cloud_powers/synapse/broadcast/broadcast.rb
+- lib/cloud_powers/synapse/pipe/pipe.rb
+- lib/cloud_powers/synapse/queue/board.rb
+- lib/cloud_powers/synapse/queue/queue.rb
 - lib/cloud_powers/synapse/synapse.rb
 - lib/cloud_powers/version.rb
 - lib/cloud_powers/workflow.rb

data/lib/cloud_powers/synapse/queue.rb

@@ -1,142 +0,0 @@
-require 'uri'
-
-module Smash
-  module CloudPowers
-    module Synapse
-      module Queue
-        include Smash::CloudPowers::Helper
-        include Smash::CloudPowers::AwsResources
-        # Board <Struct>
-        # This groups common functionality for a queue
-        Board = Struct.new(:sqs, :set_name, :set_address, :workflow) do
-          include Smash::CloudPowers::Synapse::Queue
-          include Smash::CloudPowers::Helper
-          include Smash::CloudPowers::Zenv
-
-          def i_var
-            "@#{name}"
-          end
-
-          def address
-            set_address || zfind(set_name) ||
-               "https://sqs.us-west-2.amazonaws.com/#{zfind(:account_number)}/#{name}"
-          end
-
-          def create_queue!
-            sqs.create_queue(queue_name: to_camel(name))
-            self
-          end
-
-          def destroy!
-            sqs.delete_queue(queue_url: address)
-          end
-
-          def message_count
-            get_queue_message_count(address)
-          end
-
-          def name
-            set_name || address.split('/').last
-          end
-
-          def next_board
-            workflow.next
-          end
-
-          def pluck_message(board_name = name)
-            pluck_queue_message(board_name)
-          end
-
-          def real?
-            queue_exists?(name)
-          end
-
-          def send_message(message)
-            send_queue_message(
-              address, (valid_json?(message) ? message : message.to_json)
-            )
-          end
-        end # end Board
-        #############################################
-
-        def board_name(url)
-          url.to_s.split('/').last
-        end
-
-        def create_queue!(name)
-          begin
-            Board.new(sqs, to_camel(name)).create_queue!
-          rescue Aws::SQS::Errors::QueueDeletedRecently => e
-            sleep 5
-            retry
-          end
-        end
-
-        def build_queue(name)
-          Board.new(sqs, to_camel(name))
-        end
-
-        def delete_queue_message(queue, opts = {})
-          poll(queue, opts) do |msg, stats|
-            poller(queue).delete_message(msg)
-            throw :stop_polling
-          end
-        end
-
-        def get_queue_message_count(board_url)
-          sqs.get_queue_attributes(
-            queue_url: board_url,
-            attribute_names: ['ApproximateNumberOfMessages']
-          ).attributes['ApproximateNumberOfMessages'].to_f
-        end
-
-        # @params: board<String|symbol>: The name of the board
-        # @returns a message and deletes it from its origin
-        def pluck_queue_message(board)
-          poll(board) do |msg, poller|
-            poller.delete_message(msg)
-            return valid_json?(msg.body) ? JSON.parse(msg.body) : msg.body
-          end
-        end
-
-        def poll(board, opts = {})
-          this_poller = poller(board)
-          results = nil
-          this_poller.poll(opts) do |msg|
-            results = yield msg, this_poller if block_given?
-            this_poller.delete_message(msg)
-            throw :stop_polling
-          end
-          results
-        end
-
-        def poller(board_name)
-          board = Board.new(sqs, board_name)
-
-          unless instance_variable_defined?(board.i_var)
-            instance_variable_set(
-              board.i_var,
-              Aws::SQS::QueuePoller.new(board.address)
-            )
-          end
-          instance_variable_get(board.i_var)
-        end
-
-        def queue_exists?(name)
-          !sqs.list_queues(queue_name_prefix: name).queue_urls.empty?
-        end
-
-        def queue_search(name)
-          sqs.list_queues(queue_name_prefix: name).queue_urls
-        end
-
-        def send_queue_message(address, message)
-          sqs.send_message(
-            queue_url: address,
-            message_body: message
-          )
-        end
-      end
-    end
-  end
-end