cloud_powers 1.0.1 → 1.1.0

data/lib/cloud_powers/storage/bucket.rb

@@ -0,0 +1,217 @@
+require 'cloud_powers/helpers'
+require 'cloud_powers/storage'
+
+module Smash
+  module CloudPowers
+    module Storage
+      class Bucket < Smash::CloudPowers::Resource
+        include Smash::CloudPowers::Storage
+
+        attr_accessor :origin, :delimiter, :tie_in_path
+
+        def initialize(name:, client: s3, **config)
+          super
+          @bucket = Aws::S3::Bucket.new(name: name, client: client)
+          build_storage(name: 'local', named_type: 'storage').link
+          @delimiter = config[:delimiter] || common_delimiter
+          # should be the 'lowest level' of storage this object is aware of
+          @origin = config[:origin] || project_root.split.last
+          @tie_in_path = paths_gcd(local_storage.tie_in_path, project_root)
+        end
+
+        def self.tie_in_config
+          # TODO make this able to use #to_snake so it can be dynamic
+          { name: 'bucket', client: 's3' }
+        end
+
+        # def bucket_find(pattern, **config)
+        #   bucket_select(pattern).first
+        # end
+
+        # def bucket_select(pattern, **config)
+        #   client.list_buckets.buckets.select { |b| %r"#{name}" =~ b.name }
+        # end
+
+        def create_resource
+          @response = s3.create_bucket(bucket: name)
+          yield self if block_given?
+          self
+        end
+
+        def exists?
+          # TODO nil.exists? will asplode
+          !!(@bucket.exists? rescue nil)
+        end
+
+        # Find out if a job exists in the bucket
+        #
+        # Parameters
+        # * file_name +Pathname+|+String+ - the name of the file you want to pass
+        # * location +Pathname+|+String+ (optional) - this helps speed up the
+        #   process by limiting any searching for the file
+        #
+        # Returns
+        # +IO+ -
+        def job_file_exists?(file_name, location = job_path)
+          normalized_location = paths_gcd(project_root, location)
+          !get_file_names(file_name, location: normalized_location).empty?
+        end
+
+        def link
+          if exists?
+            # do stuff
+          else
+            save!
+          end
+
+          @linked = @response.nil?
+        end
+
+        def select(*args, **opts)
+          pattern = args.pop
+          location = args.shift || origin
+
+          names = if filename?(pattern)
+            get_file_names(pattern, location: location)
+          else
+            get_bucket_names(pattern, location: location)
+          end
+
+
+          names = get_file_names(pattern, location: location)
+          if block_given?
+            names.map do |file_name|
+              new_file = to_realpath(local_storage.find(file_name))
+              File.open(new_file, 'r+') do |file|
+                data = get_file(file_name, location: location).get.body.read
+                yield file, data
+              end
+            end
+          else
+            names
+          end
+        end
+
+        def find(*args, location: origin, **opts)
+          path = select(*args, location: location, **opts).first
+          file_name = path.kind_of?(Pathname) ? path.split.last.to_s : path
+          file_permissions = opts[:file_permissions] || 'a+'
+          path = local_storage.tie_in_path + path
+
+          if block_given?
+            begin
+              File.open(path, file_permissions) do |file|
+                data = get_file(file_name, location: location).get.body.read
+                yield file, data
+              end
+            rescue Exception => e
+              if e =~ /no implicit conversion of nil into String/
+                logger.info "no file found from #{path} called #{file}"
+                nil
+              else
+                super
+              end
+            end
+          else
+            path
+          end
+        end
+
+        # Notes:
+        # * See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#put_object-instance_method
+        def put(file_name, location = origin, **config)
+          body = get_local_file_body(file_name, location)
+          return if body.nil?
+
+          client.put_object(
+            put_config(body: body, key: file_name, **config)
+          )#.on_success { Thread.new { update_registry } }
+          true
+        end
+
+        private
+        # Get a local file using the file name and optional location
+        #
+        # Parameters
+        # * +Pathname+ - absolute path is preferable but a search for the file
+        #   is initiated if it isn't found at <tt>local_storage.origin</tt>
+        #
+        # Returns
+        # +String+|+IO+
+        def get_local_file_body(file, location = origin)
+          scope = paths_gcd(tie_in_path, location)
+          local_storage.
+            find(file, location: scope, file_rights: 'r') { |data| data }
+        end
+
+        # Get object names from repeated
+        # <tt>Aws::S3::Client#list_objects_v2</tt> calls.
+        #
+        # Parameters
+        # * pattern +Regex+|+String+(glob) - the name of the object you are
+        #   looking for
+        # * config +KeywordArgument+(s) - override options for the request to
+        #   <tt>Aws::S3::Client.list_objects_v2</tt>
+        #
+        # Returns
+        # * +Array+ of Aws responses
+        #
+        # Notes:
+        # * See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Client.html#list_objects_v2-instance_method
+        # * See https://docs.aws.amazon.com/sdkforruby/api/Aws/S3/Bucket.html#objects-instance_method
+        def get_file_names(pattern, location: origin)
+          objects = @bucket.objects(
+            delimiter: delimiter,
+            encoding_type: "url",
+            marker: "Marker",
+            prefix: location.to_s,
+          ).map(&:key).grep(pattern)
+        end
+
+        def get_bucket_names(pattern, location: origin)
+          buckets = client.list_buckets.buckets.map(&:name)
+        end
+
+        # Get the actual object using S3::Bucket
+        def get_file(file_name, location: origin)
+          @bucket.object(file_name)
+        end
+
+        # Tweak the config
+        def put_config(**config)
+          {
+            acl: config[:acl] || "private", # accepts private, public-read, public-read-write, authenticated-read, aws-exec-read, bucket-owner-read, bucket-owner-full-control
+            body: config[:body], # file/IO object, or string data
+            bucket: name, # required
+            # cache_control: "CacheControl",
+            # content_disposition: "ContentDisposition",
+            # content_encoding: "ContentEncoding",
+            # content_language: "ContentLanguage",
+            # content_length: 1,
+            # content_md5: "ContentMD5",
+            # content_type: "ContentType",
+            # expires: Time.now,
+            # grant_full_control: "GrantFullControl",
+            # grant_read: "GrantRead",
+            # grant_read_acp: "GrantReadACP",
+            # grant_write_acp: "GrantWriteACP",
+            key: config[:file_name], # required
+            # metadata: {
+            #   "MetadataKey" => "MetadataValue",
+            # },
+            server_side_encryption: "AES256", # accepts AES256, aws:kms
+            storage_class: "STANDARD", # accepts STANDARD, REDUCED_REDUNDANCY, STANDARD_IA
+            # website_redirect_location: "WebsiteRedirectLocation",
+            # sse_customer_algorithm: "SSECustomerAlgorithm",
+            # sse_customer_key: "SSECustomerKey",
+            # sse_customer_key_md5: "SSECustomerKeyMD5",
+            # ssekms_key_id: "SSEKMSKeyId",
+            # request_payer: "requester", # accepts requester
+            tagging: "TaggingHeader", # !! Relate this to the project !!
+            use_accelerate_endpoint: false,
+          }.merge(config)
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/storage/local.rb

@@ -0,0 +1,129 @@
+require 'cloud_powers/storage'
+
+module Smash
+  module CloudPowers
+    module Storage
+      class Local < Smash::CloudPowers::Resource
+        include Smash::CloudPowers::Storage
+        include Smash::CloudPowers::Zenv
+
+        attr_accessor :origin, :tie_in_path
+
+        def initialize(name:, origin: zlib_home, **config)
+          super
+          @origin = origin.kind_of?(Pathname) ? origin : to_pathname(origin)
+          @tie_in_path = to_pathname(@origin, to_camel(name))
+        end
+
+        def self.tie_in_config
+          # TODO make this able to use #to_snake so it can be dynamic
+          { type: 'local', origin: 'zlib_path' }
+        end
+
+        # def self.config(*opts)
+        #   {
+        #     name: opts.select { |k| k.kind_of? String }.first || 'local',
+        #     origin: opts.select { |k| k.kind_of? Pathname }.first,
+        #     config: opts.select { |k| k.kind_of? Hash }.first
+        #   }
+        # end
+
+        # Creates an actual directory or file using the standard format for
+        # <b><i>this</i></b> storage name (camel case)
+        #
+        # Returns
+        # <tt>Storage::Local</tt?
+        def create_resource
+          @response = to_realpath(tie_in_path)
+
+          yield self if block_given?
+          self
+        end
+
+        # Find out if this local directory exists.  Because directories are one
+        # of the ways to scope in <tt>CloudPowers::Storage::Local</tt>, we can
+        # find out if this local object is already on disk by searching the
+        # system, up to and including the root directory.  The search attempts
+        # to first search through a smaller scope by checking in the
+        # +project_root+, +origin+ and +./zlib+ directories before giving up
+        # and searching the whole machine from +/+ or +C:\+ etc
+        #
+        # Parameters
+        # * scope +Pathname+ - default is +project_root+|+origin+|+zlib+|+Root Dir+
+        #
+        # Returns
+        # * +Boolean+
+        #
+        # Notes:
+        # * See <tt>#initialize</tt> for +origin+
+        def exists?(scope: origin.parent)
+          search_results = path_search(name, scope)
+          !(search_results.empty?)
+        end
+
+        # Tie the object on disk with this object, in memory.  If the object
+        # on disk doesn't exist, it will be created now.
+        #
+        # Returns
+        # * <tt>Smash::CloudPowers::Storage::Local</tt> - +self+
+        def link
+          @linked = exists? ? true : save!
+          self
+        end
+
+        def job_file_exists?(object_name = '')
+          !!(find(object_name, location: job_home))
+        end
+
+        def select(*patterns, **opts)
+          file_rights = opts[:file_rights] || 'a+'
+          scope = paths_lcd(
+            tie_in_path, to_pathname(opts[:location]).expand_path
+          )
+
+          regexs = patterns.map do |pattern|
+            if pattern.kind_of?(Pathname)
+              additional_scope, basename = pattern.split
+              scope = paths_lcd(additional_scope, scope)
+              %r"#{basename}$"
+            else
+              pattern.kind_of?(Regexp) ? pattern : %r"#{pattern.to_s}$"
+            end
+          end
+
+          paths = regexs.map do |regex|
+            Dir["#{scope}/**/*"].
+              grep(regex).
+              map { |path| to_realpath(path) }
+            end.flatten
+
+          if block_given?
+            paths.map { |path| yield File.open(path, file_rights) }
+          else
+            paths
+          end.flatten
+        end
+
+        def find(*args, location: origin, **opts)
+          # TODO:
+          # it's good to use select but this should use a stronger search or
+          # something, to provide something more than just `.first`-ing an array
+          path = select(*args, location: location, **opts).first
+          file_rights = opts[:file_rights] || 'a+'
+          if block_given?
+            begin
+              File.open(path, file_rights) do |file|
+                yield file.read
+              end
+            rescue TypeError => e
+              logger.info("file doesn't exist #{path}")
+              nil
+            end
+          else
+            path
+          end
+        end
+      end
+    end
+  end
+end

data/lib/cloud_powers/storage.rb

@@ -1,16 +1,85 @@
 require 'pathname'
-require_relative 'aws_resources'
+require 'cloud_powers/aws_resources'
+require 'cloud_powers/helpers'
+require 'cloud_powers/storage/bucket'
+require 'cloud_powers/storage/local'
 
 module Smash
   module CloudPowers
     module Storage
       include Smash::CloudPowers::AwsResources
+      include Smash::CloudPowers::Helpers
+
+      def existing_storage(types = all_storage_types)
+        normalized_types = types.map { |type| to_pascal(type) }
+        i_vars.select do |k,v|
+          normalized_types.include?(to_pascal(v.type)) rescue nil
+        end.values
+      end
+
+      # Get all existing Storage.  If there isn't any existing Storage objects,
+      # get some delegates for the available types.  If parameters are passed,
+      # narrow down the result set with the params.
+      #
+      # Parameters
+      # * queries +String+|+Symbol+|+Storage+ - a
+      #   * +String+ will represent a name and gets searched for using
+      #     delegates.  If a location is found with no Storage object that
+      #     matches, it builds it/them.
+      #   * +Symbol+s are used to narrow down the types used in the search.
+      #   * +Storage+ This method also allows you to pass in +Storage+ objects
+      #     and have them passed into the list of returned +Storage+ objects.
+      #     This is useful for allowing this method to be used in other methods
+      #
+      # Returns
+      # * +Array+ of <tt>Smash::CloudPowers::Storage[::?]</tt>
+
+      def all_storage(queries = [])
+        types, passed, names = sort_all_storage_params(queries)
+        # combine the search efforts from above
+        found = existing_storage(types) + passed
+        found_names = found.map(&:name)
+
+        # make sure all the names that were not found are represented as a new
+        # Storage object, if it isn't already
+        unmatched_names = names - found_names
+
+        # if all the names are represented, return the correct results
+        if unmatched_names.empty?
+          (found + storage_delegates(types)).uniq
+        else
+          # or build ones that should be and add it to the results, then return
+          # them all.
+          storage_delegates(types).map do |storage|
+            unlinked_storages = storage.select(*unmatched_names).reject do |n|
+              filename? n
+            end
+
+            unlinked_storages.map do |unlinked_storage|
+              # TODO create a method in path_help to separate.  it can be used
+              # here and instead of doing it in to_snake, to_snake can use it
+              # too
+              clues = unlinked_storage.to_s.split(%r"#{common_delimiter}")
+
+              config = {
+                name:         clues.pop,
+                type:         storage.type,
+                origin:       clues.join(common_delimiter)
+              }
+
+              config = storage_config(storage.type).merge(config)
+              build_storage(**config)
+            end
+          end.flatten
+        end
+      end
 
       def local_job_file_exists?(file)
-        File.exists?(job_path(to_ruby_file_name(file)))
+        path = job_path(to_ruby_file_name(file))
+        storage_select(file, location: [:local, path]).count > 0
       end
 
-      # Searches a local job storage location for the given +file+ name
+      # Searches a local jobs storage location for the given +file+ name
       # if it exists - exit the method
       # if it does <i>not</i> exist - get the file from s3 and place it in
       # the directory that was just searched bucket using +#zfind()+
@@ -43,17 +112,22 @@ module Smash
       #                   |             |_foobar.rb
       #                   |             |_custom_greetings.js # could be an after effects JS script
       def source_job(file)
-        # TODO: better path management
-        bucket = zfind('job storage')
-        if job_path(to_ruby_file_name(file)).nil?
-          objects = s3.list_objects(bucket: bucket).contents.select do |f|
-            /#{Regexp.escape file}/i =~ f.key
+        storage_delegates(:bucket).map do |delegate|
+          delegate.find(file, location: zfind('jobs storage')) do |f, data|
+            f.write data
           end
+        end.flatten
+        # TODO: better path management
+        # bucket = zfind('jobs storage')
+        # if job_path(to_ruby_file_name(file)).nil?
+        #   objects = s3.list_objects(bucket: bucket).contents.select do |f|
+        #     /#{Regexp.escape file}/i =~ f.key
+        #   end
 
-          objects.each do |obj|
-            s3.get_object(bucket: bucket, key: obj.key, response_target: job_path(file))
-          end
-        end
+        #   objects.each do |obj|
+        #     s3.get_object(bucket: bucket, key: obj.key, response_target: job_path(file))
+        #   end
+        # end
       end
 
       # Search through a bucket to find a file, based on a regex
@@ -69,6 +143,8 @@ module Smash
       #   matches.first.contents.size
       #   # => 238934 # integer representation of the file size
       def search(bucket, pattern)
+        # this guard allows methods that return nil to be used as the 'bucket',
+        # like #zfind()
         return [] if bucket.nil?
 
         begin
@@ -82,6 +158,69 @@ module Smash
         end
       end
 
+      # This method builds a <tt>CloudPowers::Storage::</tt> object for you to
+      # use but doesn't invoke the <tt>#create!()</tt> method, so no API call is
+      # made, no directories are created, etc.  This can be used even if the
+      # storage already exists.
+      #
+      # Parameters
+      # * name +String+ - name of the storage you want to interact with
+      #
+      # Returns
+      # Storage::[Local|Bucket]
+      #
+      # Example
+      #   storage_object = build_queue('exampleQueue')
+      #   storage_object.select('job')
+      #   => job.rb
+      def build_storage(name:, type: nil, **config)
+        type = to_pascal(type || :local)
+
+        storage_resource = Smash::CloudPowers::Storage.const_get(type).
+          build(name: to_camel(name), type: (type || :storage), **config)
+
+        yield storage_resource if block_given?
+
+        attr_map(storage_resource.call_name => storage_resource) do |attribute, resource|
+          instance_attr_accessor attribute
+          resource
+        end
+
+        storage_resource
+      end
+
+
+      # Create a storage without explicitly creating an object, of whatever type
+      # we're working with, here.  The types can be of any
+      # <tt>CloudPowers::Storage</tt> or any other class, as long as it follows
+      # the <tt>CloudPowers::Createable</tt> Interface
+      #
+      # Parameters
+      # * name +String+ - The name of the Queue to be created
+      #
+      # Returns
+      # CloudPowers::Storage::[Local|Bucket]
+      #
+      # Example
+      #   create_queue('exampleQueue')
+      #   get_queue_message_count
+      def create_storage(name:, type: nil, **config)
+        # default is :local
+        type, config[:type] = type.nil? ? [:local, :storage] : [type, type]
+
+        storage_resource =
+          Smash::CloudPowers::Storage.const_get(to_pascal(type)).create!(
+            name: to_camel(name), **config
+          )
+
+        attr_map(storage_resource.call_name => storage_resource) do |attribute, resource|
+          instance_attr_accessor attribute
+          resource
+        end
+
+        storage_resource
+      end
+
       # Send the log files to the S3 log file bucket
       #
       # Returns
@@ -95,6 +234,139 @@ module Smash
           )
         end
       end
+
+      # Search through all Storage or just the ones that fit your search
+      # parameters.  The searching algorithm takes you through existing
+      # +Storage+ objects and the locations they represent, like local the file
+      # system and AWS S3 Buckets
+      #
+      # Parameters
+      # * list of +String+ - the name(s) of the file, direcotry, bucket, etc you
+      #   are looking for
+      # * list of +KeyValue+ pairs - search options that are used for narrowing
+      #   down your objects
+      # * * +:location+ +:local+|+:bucket+|+String+
+      #
+      # Returns
+      # * Array +Objects+ - can be the return value of whichever Storage object
+      # found something that matched the param(s)
+      #
+      # Notes:
+      # * Narrowing down the search results speeds up the search
+      # * If a block is passed to this method, it will be run in the
+      #   <tt>#select</tt> method of the current Storage type
+      # * See the <tt>#select</tt> methods for all the Storage classes you are
+      #   using
+      def storage_select(*names, **opts)
+        # make sure to process all possible types of names that can come in.
+        # symbols are reserved for types, not names.
+        locations = [opts.delete(:location) || opts.delete(:locations)].flatten
+        locations = (locations + extract!(names) { |n| n.kind_of? Symbol }).uniq
+
+        all_storage(locations).map do |storage|
+          if block_given?
+            storage.select(*names, **opts) do |object|
+              yield object
+            end
+          else
+            storage.select(*names, **opts)
+          end
+        end.flatten
+      end
+
+      private
+      # Get all types that are both +Storage+ and work with the +Creatable+ ->
+      # +Resource+ interface.
+      #
+      # Parameters
+      # * a list of types that both fit the requirements listed in this method's
+      #   description and are in the list given as parameter(s).
+      #
+      # Returns
+      # * <tt>Array[Symbols]</tt>
+      def all_storage_types(*types)
+        # TODO - Death to harcoding!!!
+        # this should be able to query the Storage module or something to figure
+        # out all the storage types instead of hardcoding
+        [:local, :bucket] - types
+      end
+
+      # sort out the parameters that can be sent to the <tt>#all_storage</tt>
+      # method, to get rid of some of the bulk in that method.
+      #
+      # Parameters
+      # * queries +Array+ of +Symbol+|+String+|+Storage+ - see <tt>all_storage</tt>
+      #   for a description of the params
+      #
+      # Returns
+      # +Array+ of +Array+ of segregated params
+      def sort_all_storage_params(queries)
+        # get all types from the queries parameter then normalize them so they
+        # can be compared to another array's terms
+        types = queries.select { |element| element.kind_of? Symbol }
+        types = all_storage_types if types.empty?
+        normalized_types = types.map { |element| to_pascal(element) }
+
+        # get all the Storage objects and pass them through, to the returned
+        # Storage objects
+        passed = queries.select do |element|
+          normalized_types.include?(to_pascal(element.type)) rescue nil
+        end
+
+        # get all the Strings and use them as names of Storage(s)
+        names = queries.select { |element| element.kind_of? String }
+        [types, passed, names]
+      end
+
+      # Get delegate Storage objects for every type available.  A delegate is
+      # just a basic Storage object of a given type, that can access enough
+      # locations, on its own, that it can be used as a good, base representation
+      # of a Storage location, "out in the wild"; e.g. local, cloud, networked
+      # etc.
+      #
+      # Returns
+      # * +Array+ of <tt>Smash::CloudPowers::Storage[::?]</tt> - with access to
+      #   all or most of the "root" of the storage out in the wild
+      def storage_delegates(types = all_storage_types)
+        types = [types].flatten
+        types.map do |type|
+          delegate_call_name = "delegate_#{type}"
+          if respond_to? delegate_call_name
+            public_send delegate_call_name
+          else
+            config = storage_config(type).merge({ name: 'delegate', type: type })
+            # set delegates up to search everywhere
+            build_storage(**config) do |storage|
+              storage.tie_in_path = storage.origin
+            end
+          end
+        end
+      end
+
+      # Get information directly from the class you will soon instantiate.  This
+      # method gathers information from the class that the class itself deems
+      # important enough to tell you about, for instantiating it.  The method
+      # will call 0..n methods on your current class to gain information from
+      # this context; that it may be used in the new storage class' context as
+      # well.
+      #
+      # The best example of this is the clients
+      def storage_config(type)
+        tie_in_config = begin
+          Smash::CloudPowers.const_get(to_pascal(type)).
+            tie_in_config.
+              inject({}) do |conf, (attribute, method)|
+                conf[attribute] = if self.respond_to? method
+                  public_send(method)
+                else
+                  method
+                end
+                conf
+              end
+        rescue Exception => e
+          { name: to_snake(type) }
+        end
+      end
     end
   end
 end