cloud_powers 0.2.1 → 0.2.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: abb854d94798c1f90ec78dba1e944d34c978cc44
-  data.tar.gz: fcfceeba844c73fd520e614b276ddc11266dd557
+  metadata.gz: 70615a0db0f98f47026a8d05b715553490dfeb00
+  data.tar.gz: 7e2aca01ee8b38955586ae0d3fb42029d5bc9457
 SHA512:
-  metadata.gz: ead5833b4a6b7a13d079f5e2224e2e3f6a116db877ba96be4da3698e6604d7122461100efa828537a3779b6bb3ab888ccbcc1766cbfa0a62b5819e539ef42efc
-  data.tar.gz: 3c09c99ea6503ed6f53ceeeadd1c270cdc53dc4d72390590d5206af340c5ce6d47f450b717364f115556d598495e883d6ae93eb571709a35b7450bf0a5d916a9
+  metadata.gz: b1da98d073d2d3fe8af5200fecd185ef8af1cba65672aba687780dd3f1d0b5fe52110b5470467a0d8f01bc40d6fcbf2b4883780e6e60b0d7d69c6a2062edee86
+  data.tar.gz: 6850a5a5e8c354599cd6ed18d437ff28961140ebacbede6047ab9728f544f8cac0219119c31e511424491acaad8e63d3d8c818a2345cd1747091f4e0da0d75fc

data/.test.env.example

@@ -0,0 +1,27 @@
+# Project information:
+PROJECT_ROOT=""
+
+# Aws access:
+AWS_ACCESS_KEY_ID=""
+AWS_SECRET_ACCESS_KEY=""
+
+# Aws areas and auth-related locations: 
+AWS_REGION=""
+
+# Aws Build info:
+AMI_NAME="for Cerebrums to create a node"
+
+# Aws kinesis stream names
+STATUS_STREAM="e.g. kinesis stream name"
+
+# Aws s3 buckets etc
+TASK_STORAGE="e.g. s3 object name"
+
+# Aws sqs queue addresses
+BACKLOG_QUEUE_ADDRESS=""
+WIP_QUEUE_ADDRESS=""
+FINISHED_QUEUE_ADDRESS=""
+
+
+# --THIS SHOULD ALWAYS BE THE LAST LINE-- < delete everything below this line for production
+TESTING="true"

data/.travis.yml

@@ -1,5 +1,4 @@
 sudo: false
 language: ruby
-rvm:
-  - 2.2.1
+
 before_install: gem install bundler -v 1.12.5

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    cloud_powers (0.2.1)
+    cloud_powers (0.2.2)
       activesupport-core-ext (~> 4)
       aws-sdk (~> 2)
       dotenv (~> 2.1)

data/Rakefile

@@ -4,3 +4,7 @@ require "rspec/core/rake_task"
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+task :console do
+  exec "irb -r cloud_powers -I ./lib"
+end

data/cloud_powers.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
     CloudPowers is a wrapper around AWS and other cloud services.
     It was developed specifically for the Brain project
     but hopefully can be used in any other ruby project that needs to use cloud
-    service providers' resources.  Version 0.1.5 has a little EC2, S3, SQS and
+    service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS and
     Kinesis.  This project is actively being developed, so more additions, specs
     and docs will be updated frequently.  I always welcome input.
     Enjoy!

data/lib/cloud_powers/aws_resources.rb

@@ -34,11 +34,16 @@ module Smash
       end
 
       def s3
+        byebug
         @s3 ||= Aws::S3::Client.new(
           region: region,
           credentials: Auth.creds
         )
       end
+
+      def sqs
+        @sqs ||= Aws::SQS::Client.new(credentials: Auth.creds)
+      end
     end
   end
 end

data/lib/cloud_powers/helper.rb

@@ -151,6 +151,15 @@ module Smash
           extraInfo:        updated_extra_info
         }.merge(opts)
       end
+
+      def valid_json?(json)
+        begin
+          JSON.parse(json)
+          true
+        rescue Exception => e
+          false
+        end
+      end
     end
   end
 end

data/lib/cloud_powers/self_awareness.rb

@@ -12,8 +12,8 @@ module Smash
       extend Smash::CloudPowers::Helper
       extend Smash::CloudPowers::Synapse::Pipe
       extend Smash::CloudPowers::Synapse::Queue
+      extend Smash::CloudPowers::Zenv
       include Smash::CloudPowers::AwsResources
-      include Smash::CloudPowers::Zenv
 
       # Gets the instance time or the time it was called and as seconds from
       # epoch
@@ -23,7 +23,8 @@ module Smash
           @boot_time ||=
             ec2.describe_instances(dry_run: zfind('testing'), instance_ids:[@instance_id]).
               reservations[0].instances[0].launch_time.to_i
-        rescue Aws::EC2::Errors::DryRunOperation => e
+        # rescue Aws::EC2::Errors::DryRunOperation => e
+        rescue Exception => e
           logger.info "dry run for testing: #{e}"
           @boot_time ||= Time.now.to_i # comment the code below for development mode
         end
@@ -74,8 +75,10 @@ module Smash
         # get @task_name
         return @task_name unless @task_name.nil?
         # set @task_name
-        resp = ec2.describe_instances(instance_ids: [id].flatten)
-        @task_name = resp.reservations[0].instances[0].tags.select do |t|
+        # TODO: get all tasks instead of just the first
+        resp = ec2.describe_instances(instance_ids: [id].flatten).reservations.first
+        return @task_name = nil if resp.nil?
+        @task_name = resp.instances[0].tags.select do |t|
           t.value if t.key == 'taskType'
         end.first
       end

data/lib/cloud_powers/synapse/queue.rb

@@ -1,22 +1,38 @@
 require 'uri'
-require_relative '../helper'
 
 module Smash
   module CloudPowers
     module Synapse
-      include Smash::CloudPowers::Helper
-
       module Queue
-        Board = Struct.new(:set_name, :set_address, :workflow) do
+        include Smash::CloudPowers::Helper
+        include Smash::CloudPowers::AwsResources
+        # Board <Struct>
+        # This groups common functionality for a queue
+        Board = Struct.new(:sqs, :set_name, :set_address, :workflow) do
+          include Smash::CloudPowers::Synapse::Queue
           include Smash::CloudPowers::Helper
+          include Smash::CloudPowers::Zenv
+
           def i_var
             "@#{name}"
           end
 
           def address
-            set_address ||
-              env(name) ||
-              "https://sqs.us-west-2.amazonaws.com/088617881078/#{name}"
+            set_address || zfind(set_name) ||
+               "https://sqs.us-west-2.amazonaws.com/#{zfind(:account_number)}/#{name}"
+          end
+
+          def create_queue!
+            sqs.create_queue(queue_name: to_camel(name))
+            self
+          end
+
+          def destroy!
+            sqs.delete_queue(queue_url: address)
+          end
+
+          def message_count
+            get_queue_message_count(address)
           end
 
           def name
@@ -26,6 +42,20 @@ module Smash
           def next_board
             workflow.next
           end
+
+          def pluck_message(board_name = name)
+            pluck_queue_message(board_name)
+          end
+
+          def real?
+            queue_exists?(name)
+          end
+
+          def send_message(message)
+            send_queue_message(
+              address, (valid_json?(message) ? message : message.to_json)
+            )
+          end
         end # end Board
         #############################################
 
@@ -33,19 +63,17 @@ module Smash
           url.to_s.split('/').last
         end
 
-        # def board_name(url)
-        #   # TODO: figure out a way to not have this and :name in Board
-        #   # gets the name from the url
-        #   if url =~ URI.regexp
-        #     url = URI.parse(url)
-        #     url.path.split('/').last.split('_').last
-        #   else
-        #     env(url)
-        #   end
-        # end
-
-        def create_queue(name)
-          sqs.create_queue(queue_name: to_camel(name))
+        def create_queue!(name)
+          begin
+            Board.new(sqs, to_camel(name)).create_queue!
+          rescue Aws::SQS::Errors::QueueDeletedRecently => e
+            sleep 5
+            retry
+          end
+        end
+
+        def build_queue(name)
+          Board.new(sqs, to_camel(name))
         end
 
         def delete_queue_message(queue, opts = {})
@@ -55,31 +83,35 @@ module Smash
           end
         end
 
-        def get_count(board)
+        def get_queue_message_count(board_url)
           sqs.get_queue_attributes(
-            queue_url: board_name(board),
+            queue_url: board_url,
             attribute_names: ['ApproximateNumberOfMessages']
           ).attributes['ApproximateNumberOfMessages'].to_f
         end
 
-        # Params: board<string>
-        # returns a message and deletes it from its origin
-        def pluck_message(board)
+        # @params: board<String|symbol>: The name of the board
+        # @returns a message and deletes it from its origin
+        def pluck_queue_message(board)
           poll(board) do |msg, poller|
             poller.delete_message(msg)
-            return msg
+            return valid_json?(msg.body) ? JSON.parse(msg.body) : msg.body
           end
         end
 
         def poll(board, opts = {})
           this_poller = poller(board)
+          results = nil
           this_poller.poll(opts) do |msg|
-            yield msg, this_poller if block_given?
+            results = yield msg, this_poller if block_given?
+            this_poller.delete_message(msg)
+            throw :stop_polling
           end
+          results
         end
 
         def poller(board_name)
-          board = Board.new(board_name)
+          board = Board.new(sqs, board_name)
 
           unless instance_variable_defined?(board.i_var)
             instance_variable_set(
@@ -91,21 +123,19 @@ module Smash
         end
 
         def queue_exists?(name)
-          sqs.list_queues(queue_name_prefix: name)
+          !sqs.list_queues(queue_name_prefix: name).queue_urls.empty?
+        end
+
+        def queue_search(name)
+          sqs.list_queues(queue_name_prefix: name).queue_urls
         end
 
-        def send_queue_message(message, *board_info)
-          board = board_info.first.kind_of?(Board) ? board_info.first : Board.new(*board_info)
-          message = message.to_json unless message.kind_of? String
+        def send_queue_message(address, message)
           sqs.send_message(
-            queue_url: board.address,
+            queue_url: address,
             message_body: message
           )
         end
-
-        def sqs
-          @sqs ||= Aws::SQS::Client.new(credentials: Auth.creds)
-        end
       end
     end
   end

data/lib/cloud_powers/version.rb

@@ -1,3 +1,3 @@
 module CloudPowers
-  VERSION = "0.2.1"
+  VERSION = "0.2.2"
 end

data/lib/cloud_powers/zenv.rb

@@ -7,21 +7,28 @@ module Smash
     # will also use elasticache/redis...some other stuff too, in the coming weeks
     module Zenv
       include Smash::CloudPowers::Helper
-      # ZFind looks for the key in a preditermined order of importance:
-      #   * i-vars are considered first becuase they might be tracking different
-      #     locations for multiple tasks or something like that.
-      #   * dotenv files are second because they were manually set, so for sure
-      #     it's important
-      #   * System Env[@] variables are up next.  Hopefully by this time we've found
-      #     our information but if not, it should "search" through the system env too.
-      # @params: key <String>: The key to search for
-      # @return: <String>
-      #   TODO: implement a search for all 3 that can find close matches
-      def zfind(key)
-        res = (i_vars[to_snake(key).upcase] or
-          env_vars[to_snake(key).upcase] unless @project_root.nil?) or
-          system_vars[to_snake(key).upcase]
-        (res.nil? or res.empty?) ? nil : res
+
+      # Attempts to find a file by searching the current directory for the file
+      # then walking up the file tree and searching at each stop
+      # @param: name <String>: name of the file or directory to find
+      # @return: Pathname: path to the file or directory given as the `name` param
+      def file_tree_search(name)
+        next_dir = Pathname.new(`pwd`.strip).parent
+        current_dir = Pathname.new(`pwd`.strip)
+        until(next_dir == current_dir) do
+          path = Dir.glob("#{current_dir}/#{name}").first
+          return current_dir unless path.nil?
+          current_dir = next_dir
+          next_dir = next_dir.parent
+        end
+        return nil
+      end
+
+      # Search through the .env variables for a key or if no key is given,
+      # return all the .env-vars and their values
+      def env_vars(key = '')
+        return ENV if key.empty?
+        ENV[to_snake(key).upcase]
       end
 
       # Search through the instance variables for a key or if no key is given,
@@ -37,11 +44,27 @@ module Smash
         self.instance_variable_get(to_i_var(key))
       end
 
-      # Search through the .env variables for a key or if no key is given,
-      # return all the .env-vars and their values
-      def env_vars(key = '')
-        return ENV if key.empty?
-        ENV[to_snake(key).upcase]
+      # PROJECT_ROOT should be set as early as possible in this Node's initilize
+      # method.  This method tries to search for it, using #zfind() and if a `nil`
+      # result is returned from that search, `pwd` is used as the PROJECT_ROOT.
+      # @return: Path to the project root or where ever `pwd` resolves to <Pathname>
+      # TODO: improve this...it needs to find the gem's method's caller's project
+      # root or at least the gem's method's caller's file's location.
+      def project_root
+        if @project_root.nil?
+          file_home = Pathname.new(
+            caller_locations.first.path.strip.split(/\//).last).realdirpath.parent
+          # path = (zfind('PROJECT_ROOT') or file_home)
+          @project_root = Pathname.new(file_home)
+        end
+        @project_root
+      end
+
+      # Manually set the `@project_root` i-var as a `Pathname` object.
+      # @param: New path to the project root <String|Pathname>
+      # @return: @project_root <Pathname>
+      def project_root=(var)
+        @project_root = Pathname.new(var)
       end
 
       # Search through the system environment variables for a key or if no key
@@ -68,26 +91,22 @@ module Smash
         end
       end
 
-      # PROJECT_ROOT should be set as early as possible in this Node's initilize
-      # method.  This method tries to search for it, using #zfind() and if a `nil`
-      # result is returned from that search, `pwd` is used as the PROJECT_ROOT.
-      # @return: Path to the project root or where ever `pwd` resolves to <Pathname>
-      # TODO: improve this...it needs to find the gem's method's caller's project
-      # root or at least the gem's method's caller's file's location.
-      def project_root
-        if @project_root.nil?
-          pr = (PROJECT_ROOT rescue nil) or zfind('PROJECT_ROOT') or system('pwd')
-          @project_root = Pathname.new(pr)
-        else
-          @project_root
-        end
-      end
-
-      # Manually set the `@project_root` i-var as a `Pathname` object.
-      # @param: New path to the project root <String|Pathname>
-      # @return: @project_root <Pathname>
-      def project_root=(var)
-        @project_root = Pathname.new(var)
+      # ZFind looks for the key in a preditermined order of importance:
+      #   * i-vars are considered first becuase they might be tracking different
+      #     locations for multiple tasks or something like that.
+      #   * dotenv files are second because they were manually set, so for sure
+      #     it's important
+      #   * System Env[@] variables are up next.  Hopefully by this time we've found
+      #     our information but if not, it should "search" through the system env too.
+      # @params: key <String>: The key to search for
+      # @return: <String>
+      #   TODO: implement a search for all 3 that can find close matches
+      def zfind(key)
+        project_root if @project_root.nil?
+        res = (i_vars[to_snake(key).upcase] or
+          env_vars[to_snake(key).upcase] unless @project_root.nil?) or
+          system_vars[to_snake(key).upcase]
+        (res.nil? or res.empty?) ? nil : res
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: cloud_powers
 version: !ruby/object:Gem::Version
-  version: 0.2.1
+  version: 0.2.2
 platform: ruby
 authors:
 - Adam Phillipps
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-26 00:00:00.000000000 Z
+date: 2016-09-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport-core-ext
@@ -154,7 +154,7 @@ description: |2
       CloudPowers is a wrapper around AWS and other cloud services.
       It was developed specifically for the Brain project
       but hopefully can be used in any other ruby project that needs to use cloud
-      service providers' resources.  Version 0.1.5 has a little EC2, S3, SQS and
+      service providers' resources.  Version 0.2.3 has a little EC2, S3, SQS and
       Kinesis.  This project is actively being developed, so more additions, specs
       and docs will be updated frequently.  I always welcome input.
       Enjoy!
@@ -166,6 +166,7 @@ files:
 - ".gitignore"
 - ".rspec"
 - ".ruby-version"
+- ".test.env.example"
 - ".travis.yml"
 - Gemfile
 - Gemfile.lock