cloud_powers 0.2.7.23 → 1.0.0

data/lib/cloud_powers/helpers/path_help.rb

@@ -0,0 +1,90 @@
+require 'fileutils'
+require 'pathname'
+require 'uri'
+
+module Smash
+  module CloudPowers
+    module PathHelp
+
+      # Gives a common home for jobs to live so they can be easily grouped and
+      # found.  This method will create nested directories, based on the
+      # <tt>#project_root()</tt> method and an additional 'lib/jobs' directory.
+      # If no project root has been set by the time this method is called, a new
+      # directory will be created relative to the gem's project root.
+      #
+      # Returns
+      # +String+
+      #
+      # Notes
+      # * # If no project root has been set by the time this method is called, a new
+      # directory will be created relative to the gem's project root.  This might
+      # have deeper implications than you want to deal with so it's always a good
+      # idea to set your project root as soon as you can.
+      # * TODO: find a way to have this method figure out the actual project's
+      #   root, as opposed to just making common <i>"good"</i> assumptions.
+      def job_home
+        string_th = FileUtils.mkdir_p("#{project_root}/lib/jobs/").first
+        @job_home ||= Pathname.new(string_th).realpath.to_s
+      end
+
+      # Gives the path from the project root to lib/jobs[/#{file}.rb]
+      #
+      # Parameters
+      # * file +String+ (optional) (default is '') - name of a file
+      #
+      # Returns
+      # * path/file +String+ if +file+ parameter is given.  return has
+      #   '.rb' extension included
+      # * file +String+ if +file+ parameter is not given it will return the
+      #   <tt>#job_require_path()</tt>
+      #
+      # Notes
+      # * See <tt>#job_home</tt>
+      def job_path(file = '')
+       return job_home if file.empty?
+        Pathname.new("#{job_home}/#{file}").to_s
+      end
+
+
+      # Check if the job file exists in the job directory
+      #
+      # Parameters
+      # * file +String+
+      #
+      # Returns
+      # +Boolean+
+      #
+      # Notes
+      # * See +#job_home()+
+      def job_exist?(file)
+        begin
+          File.new("#{job_home}/#{file}")
+          true
+        rescue Errno::ENOENT
+          false
+        end
+      end
+
+      # Gives the path from the project root to lib/jobs[/file]
+      #
+      # Parameters String (optional)
+      # * file_name name of a file
+      #
+      # Returns
+      # * path/file +String+ if +file_name+ was given
+      # * path to job_directory if +file_name+ was <i>not</i> given
+      #
+      # Notes
+      # * Neither path nor file will have a file extension
+      # * See <tt>#job_home</tt>
+      def job_require_path(file_name = '')
+        begin
+          file_sans_extension = File.basename(file_name, '.*')
+          (Pathname.new(job_home) + file_sans_extension).to_s
+        rescue Errno::ENOENT
+          nil
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/node.rb

@@ -1,36 +1,41 @@
-require_relative 'auth'
-require_relative 'helper'
-require_relative 'self_awareness'
-require_relative 'zenv'
+require 'cloud_powers/auth'
+require 'cloud_powers/helpers'
+require 'cloud_powers/zenv'
+require 'cloud_powers/node/instance'
 
 module Smash
   module CloudPowers
     module Node
       include Smash::CloudPowers::Auth
-      include Smash::CloudPowers::Helper
-      include Smash::CloudPowers::SelfAwareness
+      include Smash::CloudPowers::Helpers
       include Smash::CloudPowers::Zenv
 
-      # These are sensible defaults that can be overriden by providing a Hash as a param.
+      # This method adds certain tags to an array of resource ids.
       #
       # Parameters
-      # * opts +Hash+ (optional)
-      #   the opts Hash should have values that should be used instead of the default
-      #   configuration.
-      def node_config(opts = {})
-        {
-          dry_run:                                zfind(:testing) || false,
-          image_id:                               image('crawlbotprod').image_id, # image(:neuron).image_id
-          instance_type:                          't2.nano',
-          min_count:                              opts[:max_count] || 0,
-          max_count:                              0,
-          key_name:                               'crawlBot',
-          security_groups:                        ['webCrawler'],
-          security_group_ids:                     ['sg-940edcf2'],
-          placement:                              { availability_zone: 'us-west-2c' },
-          disable_api_termination:                'false',
-          instance_initiated_shutdown_behavior:   'terminate'
-        }.merge(opts)
+      # * ids +Array+|+String+ - an Array or a single instance id, as an Array
+      #   of Strings or a single String
+      # * tags +Array+ - an Array of key, value hash
+      #
+      # Returns
+      # * Returns an empty response.
+      #
+      # Examples
+      #   create_tag('ami-2342354', tags: { key: "stack", value: "production"})
+      #   or
+      #   create_tag(['ami-2432342'], tags: [{ key: 'stack', value: 'production' }])
+      #   or any permutation of those
+      def batch_tag(ids, tags, client: ec2)
+        tags_opts = { resources: ids, tags: tags }
+        ec2.create_tags(tags_opts)
+        logger.info "tags for #{ids} created"
+      end
+
+      def build_node(name:, client: ec2, **config)
+        resp = Smash::CloudPowers::Node::Instance.build(name: name, client: ec2, **config)
+        i_var_name = "#{name}_node"
+        instance_attr_accessor i_var_name
+        instance_variable_set(to_i_var(i_var_name), resp)
       end
 
       # Uses +Aws::EC2#run_instances()+ to create nodes (Neurons or Cerebrums), at
@@ -43,58 +48,54 @@ module Smash
       # * opts +Hash+ (optional)
       #   an optional instance configuration hash can be passed, which will override
       #   the values in the default configuration returned by #instance_config()
-      def spin_up_neurons(opts = {}, tags = [])
-        should_wait = opts.delete(:wait) || true
-        ids = nil
-        begin
-
-          response = ec2.run_instances(node_config(opts))
-          ids = response.instances.map(&:instance_id)
-
-          if should_wait
-            count = 0
-            begin
-              ec2.wait_until(:instance_running, instance_ids: ids) do
-                logger.info "waiting for #{ids.count} Neurons to start..."
-              end
-            rescue Aws::Waiters::Errors::WaiterFailed => e
-              # TODO: deal with failed instances
-              # redo unless (count += 1 <=3 )
-            end
-          end
-
-          batch_tag(ids, tags) unless tags.empty?
-          ids
-
-        rescue Aws::EC2::Errors::DryRunOperation
-          ids = (1..(opts[:max_count] || 0)).to_a.map { |n| n.to_s }
-          logger.info "waiting for #{ids.count} Neurons to start..."
-        end
-
-        ids
+      def create_node(name:, client: ec2, **config)
+        resp = Smash::CloudPowers::Node::Instance.create!(name: name, client: ec2, **config)
+        i_var_name = "#{name}_node"
+        instance_attr_accessor i_var_name
+        instance_variable_set(to_i_var(i_var_name), resp)
       end
 
-      # This method adds certain tags to an array of resource ids.
+      # Uses +Aws::EC2#run_instances()+ to create nodes (Neurons or Cerebrums),
+      # at a rate of 0..(n <= 100) at a time, until the required number of
+      # instances has been started.  The #instance_config() method is used to
+      # create instance configuration for the #run_instances method by using
+      # the opts hash that was provided as a parameter.
       #
       # Parameters
-      # * ids +Array+|+String+ - an Array or a single instance id, as an Array of Strings or a single String
-      # * tags +Array+ - an Array of key, value hash
+      # * opts +Hash+ (optional) - an optional instance configuration hash can
+      #   be passed, which will override the values in the default configuration
+      #   returned by <tt>#instance_config()</tt>
       #
       # Returns
-      # * Returns an empty response.
-      #
-      # Examples
-      #   create_tag('ami-2342354', tags: { key: "stack", value: "production"})
-      #   or
-      #   create_tag(['ami-2432342'], tags: [{ key: 'stack', value: 'production' }])
-      #   or any permutation of those
-
-      def batch_tag(ids, tags)
-        tags_opts = { resources: ids, tags: tags }
-        ec2.create_tags(tags_opts)
-        logger.info "tags for #{ids} created"
+      # +Array+ of responses from
+      def create_nodes(name:, client: ec2, **config)
+        resp = ec2.run_instances(node_config(config)).instances
+        i_var_name = "#{name}_nodes"
+        instance_attr_accessor i_var_name
+        instance_variable_set(to_i_var(i_var_name), resp)
       end
 
+      # These are sensible defaults that can be overriden by providing a Hash as a param.
+      #
+      # Parameters
+      # * opts +Hash+ (optional)
+      #   the opts Hash should have values that should be used instead of the default
+      #   configuration.
+      def node_config(**config)
+        {
+          dry_run:                                zfind(:testing) || false,
+          image_id:                               image(zfind(:node_image)).image_id, # image(:neuron).image_id
+          instance_type:                          't2.nano',
+          min_count:                              config[:max_count] || 0,
+          max_count:                              0,
+          key_name:                               'crawlBot',
+          security_groups:                        ['webCrawler'],
+          security_group_ids:                     ['sg-940edcf2'],
+          placement:                              { availability_zone: 'us-west-2c' },
+          disable_api_termination:                'false',
+          instance_initiated_shutdown_behavior:   'terminate'
+        }.merge(config)
+      end
     end
   end
 end

data/lib/cloud_powers/node/instance.rb

@@ -0,0 +1,52 @@
+require 'cloud_powers/node'
+require 'cloud_powers/resource'
+
+module Smash
+  module CloudPowers
+    module Node
+      class Instance < Smash::CloudPowers::Resource
+        include Smash::CloudPowers::Node
+
+        # The name of the Aws::EC2 instance
+        attr_accessor :name
+        # An Aws::EC2::Client. See <tt>Smash::CloudPowers::AwsResources#ec2()</tt>
+        attr_accessor :ec2
+
+        def initialize(name:, client: ec2, **config)
+          super(name: name)
+          @ec2 = client
+        end
+
+        # Uses +Aws::EC2#run_instances()+ to create nodes (Neurons or Cerebrums), at
+        # a rate of 0..(n <= 100) at a time, until the required number of instances
+        # has been started.  The #instance_config() method is used to create instance
+        # configuration for the #run_instances method by using the opts hash that was
+        # provided as a parameter.
+        #
+        # Parameters
+        # * opts +Hash+ (optional)
+        #   an optional instance configuration hash can be passed, which will override
+        #   the values in the default configuration returned by #instance_config()
+        def create_resource
+          # response = ec2.run_instances(
+          #   node_config(max_count: 1, self.to_h)
+          # ).instances.first
+
+          instance_attr_accessor response
+          # id = @response[:instance_id]
+          begin
+            ec2.wait_until(:instance_running, instance_ids: [id]) do
+              logger.info "waiting for #{ids.count} Neurons to start..."
+            end
+          rescue Aws::Waiters::Errors::WaiterFailed => e
+            # TODO: retry stuff
+            # redo unless (count += 1 <=3 )
+          end
+
+          yield self if block_given?
+          self
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/resource.rb

@@ -0,0 +1,44 @@
+require 'cloud_powers/aws_resources'
+require 'cloud_powers/creatable'
+require 'cloud_powers/helpers'
+require 'cloud_powers/zenv'
+
+module Smash
+  module CloudPowers
+    class Resource
+      include Smash::CloudPowers::Creatable
+      include Smash::CloudPowers::AwsResources
+      include Smash::CloudPowers::Helpers
+      include Smash::CloudPowers::Zenv
+
+      # client used to make HTTP requests to the cloud
+      attr_accessor :client
+      # the name this resource can be set and retrieved by
+      attr_accessor :call_name
+      # the given name for this resource
+      attr_accessor :name
+      # whether or not a call has been made to the cloud to back this resource
+      attr_accessor :linked
+      # the ID in the cloud; e.g. ARN for AWS, etc
+      attr_accessor :remote_id
+      # tracking on the remote resource that maps to this resource
+      attr_accessor :tags
+      # the type of resource this was instantiated as
+      attr_accessor :type
+
+      # Usually this method is called by calling +super+ in another class that
+      # inherits from this class.  The initialize method follows the method
+      # signature for the active record-like pattern being followed throughout
+      # the code
+      def initialize(name:, client: nil, **config)
+        @linked = false
+        @saved = false
+        @client = client
+        @type = to_snake(self.class.name.split('::').last)
+        @call_name = to_snake("#{name}_#{@type}")
+        @name = name
+        @tags = Array.new
+      end
+    end
+  end
+end

data/lib/cloud_powers/storage.rb

@@ -6,7 +6,11 @@ module Smash
     module Storage
       include Smash::CloudPowers::AwsResources
 
-      # Searches a local task storage location for the given +file+ name
+      def local_job_file_exists?(file)
+        File.exists?(job_path(to_ruby_file_name(file)))
+      end
+
+      # Searches a local job storage location for the given +file+ name
       # if it exists - exit the method
       # if it does <i>not</i> exist - get the file from s3 and place it in
       # the directory that was just searched bucket using +#zfind()+
@@ -22,31 +26,32 @@ module Smash
       #     project_root- |
       #                   |_sub_directory
       #                   |              |_current_file.rb
-      #                   |_task_storage
+      #                   |_job_storage
       #                   |             |_demorific.rb
       #                   |             |_foobar.rb
       # * code:
-      #     source_task('demorific')
+      #     source_job('demorific')
       #     # file tree doesn't change because this file exists
-      #     source_task('custom_greetings')
+      #     source_job('custom_greetings')
       #     # file tree now looks like this
       # * new file tree
       #     project_root- |
       #                   |_sub_directory
       #                   |              |_current_file.rb
-      #                   |_task_storage
+      #                   |_job_storage
       #                   |             |_demorific.rb
       #                   |             |_foobar.rb
       #                   |             |_custom_greetings.js # could be an after effects JS script
-      def source_task(file)
+      def source_job(file)
         # TODO: better path management
-        bucket = zfind('task storage')
-        if task_path(to_ruby_file_name(file)).nil?
+        bucket = zfind('job storage')
+        if job_path(to_ruby_file_name(file)).nil?
           objects = s3.list_objects(bucket: bucket).contents.select do |f|
             /#{Regexp.escape file}/i =~ f.key
           end
+
           objects.each do |obj|
-            s3.get_object(bucket: bucket, key: obj.key, response_target: task_path(file))
+            s3.get_object(bucket: bucket, key: obj.key, response_target: job_path(file))
           end
         end
       end
@@ -55,17 +60,25 @@ module Smash
       #
       # Parameters
       # * bucket +String+ - the bucket to search through in AWS
-      # * pattern +Regex+ - the Regex pattern you want to use for the
-      #   search
+      # * pattern +Regexp+|+nil+ - the Regex pattern you want to use for the
+      #   search.  nil doesn't cause an exception but probably returns <tt>[]</tt>
       #
       # Example
-      #   matches = search('neuronTasks', /[Dd]emo\w*/)
+      #   matches = search('neuronJobs', /[Dd]emo\w*/)
       #   # => ['Aws::S3::Type::ListObjectOutPut','Aws::S3::Type::ListObjectOutPut',...] # anything that matched that regex
       #   matches.first.contents.size
       #   # => 238934 # integer representation of the file size
       def search(bucket, pattern)
-        s3.list_objects(bucket: bucket).contents.select do |o|
-          o.key =~ pattern
+        return [] if bucket.nil?
+
+        begin
+          pattern = /#{pattern}/ unless pattern.kind_of? Regexp
+          s3.list_objects(bucket: bucket).contents.select do |o|
+            o.key =~ pattern || /#{o.key}/ =~ pattern.to_s
+          end
+        rescue Aws::S3::Errors::NoSuchBucket => e
+          logger.info format_error_message e
+          return # log that the bucket doesn't exist but don't explode
         end
       end
 

data/lib/{stubs → cloud_powers/stubs}/aws_stubs.rb

@@ -74,7 +74,7 @@ module Smash
       #   for your own custom configuration
       def self.node_stub(opts = {})
         time = opts[:launch_time] || Time.new((Time.now.utc.to_i / 86400 * 86400)) # midnight
-        tags = [{key: 'task', value: 'test'}]
+        tags = [{key: 'job', value: 'test'}]
         {
           stub_responses: {
             create_tags: {},
@@ -143,18 +143,22 @@ module Smash
         stub.merge(opts.select { |k,v| stub.key? k })
       end
 
-      # Get or create an Kinesis client and cache that client so that a Context is more well tied together
+      # Get or create an Kinesis client and cache that client so that a Context
+      # is more well tied together
       # Parameters
       # * opts <tt>Hash</tt>
-      # * * stub_responses: defaulted to false but it can be overriden with the desired responses for local testing
-      # * * region: defaulted to use the `#region()` method
-      # * * AWS::Credentials object, which will also scour the context and environment for your keys
+      #   * stub_responses: defaulted to false but it can be overriden with the
+      #     desired responses for local testing
+      #   * region: defaulted to use the `#region()` method
+      #   * AWS::Credentials object, which will also scour the context and
+      #     environment for your keys
       #
       # Returns
       # AWS::Kinesis client
       #
       # Example
-      #   # sets and gets an Kinesis client.  No need to set a variable because one was
+      #   # sets and gets an Kinesis client.  No need to set a variable because
+      #     one was
       #   # just created as +@kinesis+ and is set to the client
       #   kinesis(Smash::CloudPowers::AwsStubs.pipe_stub)
       #
@@ -175,6 +179,7 @@ module Smash
             },
             describe_stream: {
               stream_description: {
+                stream_creation_timestamp: Time.now - 10,
                 stream_name: opts[:name] || 'testPipe',
                 stream_arn:  'arnarnarnarnar',
                 stream_status: 'ACTIVE',
@@ -223,7 +228,25 @@ module Smash
       def self.storage_stub(opts = {})
         {
           stub_responses: {
-            head_bucket: {}
+            head_bucket: {
+
+            },
+            list_objects: {
+              is_truncated: false,
+              contents: [
+                key: 'testinz',
+                last_modified: Time.now,
+                etag: 'asdf1234',
+                size: 123,
+                storage_class: 'STANDARD',
+                owner: {display_name: 'snargle', id: 'id-bargle132'}
+              ],
+              name: 'testFile',
+              prefix: 'test',
+              delimiter: ' ',
+              max_keys: 1234,
+              common_prefixes: [{ prefix: 'test' }]
+            }
           }
         }
       end
@@ -250,14 +273,14 @@ module Smash
       #   for your own custom configuration
       def self.queue_stub(opts = {})
         msg_body = if opts[:body]
-          if opts[:body].kind_of? Hash
-            opts[:body] = opts[:body].to_json
-          elsif JSON.parse(opts[:body])
-            begin
-              opts[:body]
-            rescue JSON::ParserError
-              {foo: 'bar'}.to_json
+          begin
+            if opts[:body].kind_of? Hash
+              opts[:body] = opts[:body].to_json
+            elsif JSON.parse(opts[:body])
+                opts[:body]
             end
+          rescue JSON::ParserError
+            {foo: 'bar'}.to_json
           end
         end
         {