cifrado 0.1

data/lib/cifrado/file_splitter.rb

@@ -0,0 +1,67 @@
+module Cifrado
+
+  class FileSplitter
+    include Utils
+
+    attr_reader :cache_dir
+    attr_accessor :chunk_suffix
+
+    # FileSplitter.new("elfari.webm", 10).split
+    #
+    # @param [String] file path to split
+    # @param [Integer] number of chunks
+    # @param [String] Destination directory of the chunks
+    def initialize(filename, 
+                   chunk_number = nil, 
+                   cache_dir = File.join(ENV['HOME'], '.cache/cifrado') )
+      raise ArgumentError.new unless File.exist?(filename)
+      @filename = File.basename filename
+      @source = filename
+      @chunk_number = chunk_number || calculate_chunks(filename)
+      
+      # when we are splitting a file into a given number of chunks, 
+      # the last chunk could be bigger than the others
+      @each_size, @extra = File.size(filename).divmod(@chunk_number)
+      @cache_dir = File.expand_path cache_dir
+      @chunk_suffix = "/segments/#{'%.2f' % File.mtime(filename).to_f}/#{File.size(filename)}/"
+    end
+
+    def split
+      file = File.new(@source, "rb")
+      # create cache directory
+      unless File.directory?(@cache_dir)
+        Log.debug "Creating cache dir: #{@cache_dir}"
+        FileUtils.mkdir_p(@cache_dir) 
+      end
+
+      chunks = []
+      byte_count = 0
+      chunk_n = 1
+      chunk_name = @filename + "#{@chunk_suffix.gsub('/','-')}"
+      chunk = File.join(@cache_dir, "#{chunk_name}#{'%08d' % chunk_n}")
+      chunk_f = File.open(chunk, "wb")
+      while read = file.read(4096) do
+        if byte_count >= @each_size
+          chunk_f.close
+          chunks << chunk
+          yield chunk_n, chunk if block_given?
+          chunk_n += 1
+          chunk = File.join(@cache_dir, "#{chunk_name}#{'%08d' % chunk_n}")
+          chunk_f = File.open(chunk, "wb")
+          byte_count = 0
+        end
+        chunk_f << read
+        byte_count += 4096
+      end
+      chunk_f.flush
+      yield chunk_n, chunk if block_given?
+      chunks << chunk
+      chunks
+    ensure
+      chunk_f.close unless chunk_f.closed?
+      file.close unless file.closed?
+    end
+
+  end
+
+end

data/lib/cifrado/rate_limit.rb

@@ -0,0 +1,23 @@
+module Cifrado
+  class RateLimit
+
+    def initialize(bwlimit)
+      @time = Time.now.to_f 
+      @read = 0 
+      @sleep_time = 0.01 
+      @bwlimit = bwlimit
+    end
+
+    def limit(read)
+      bps = @read/(Time.now.to_f - @time)
+      if bps > @bwlimit
+        Log.debug 'limiting rate'
+        sleep @sleep_time
+        @sleep_time += 0.01
+      else
+        @sleep_time -= 0.01 if @sleep_time >= 0.02
+      end
+      @read += read
+    end
+  end
+end

data/lib/cifrado/streaming_downloader.rb

@@ -0,0 +1,49 @@
+require 'net/http'
+require 'net/https'
+
+module Cifrado
+  class StreamingDownloader
+
+    def self.get url, output, options = {}
+      if output.nil? and !options[:stream]
+        raise ArgumentError.new('Invalid output file')
+      end
+      uri = URI.parse url
+      http = Net::HTTP.new(uri.host, uri.port)
+      http.use_ssl = true if uri.scheme == "https"
+      copts = options[:connection_options]
+      if copts[:ssl_verify_peer] == false
+        Log.debug "SSL verification DISABLED"
+        http.verify_mode = OpenSSL::SSL::VERIFY_NONE 
+      end
+      #http.open_timeout = 10 # seconds
+      #http.read_timeout = 10 # seconds
+      Log.debug "Request URL #{uri.request_uri}"
+      request = Net::HTTP::Get.new(uri.request_uri)
+
+      headers = options[:headers]
+      request.initialize_http_header headers
+
+      rate_limit = nil
+      rate_limit = Cifrado::RateLimit.new(options[:bwlimit]) if options[:bwlimit]
+
+      unless options[:stream]
+        file = File.open(output, "wb") 
+        Log.debug "Downloading file to #{output}"
+      end
+      callback = options[:progress_callback]
+
+      http.request(request) do |response|
+        clength = response['Content-Length'].to_i
+        response.read_body do |segment|
+          rate_limit.limit(segment.size) if rate_limit
+          callback.call(clength, segment.length, segment) if callback
+          file.write(segment) if file
+        end
+      end
+    ensure
+      file.close if file
+    end
+
+  end
+end

data/lib/cifrado/streaming_uploader.rb

@@ -0,0 +1,57 @@
+module Cifrado 
+
+  # StreamingUploader Adapted by Sergio Rubio <rubiojr@frameos.org>
+  #
+  # inspired by Opscode Chef StreamingCookbookUploader chef/streaming_cookbook_uploader.rb
+  # http://opscode.com
+  # 
+  # inspired by/cargo-culted from http://stanislavvitvitskiy.blogspot.com/2008/12/multipart-post-in-ruby.html
+  # On Apr 6, 2010, at 3:00 PM, Stanislav Vitvitskiy wrote:
+  #
+  # It's free to use / modify / distribute. No need to mention anything. Just copy/paste and use.
+  #
+  # Regards,
+  # Stan
+
+
+  require 'net/http'
+  require 'net/https'
+  require 'cifrado/rate_limit'
+
+  class StreamingUploader
+
+    class << self
+
+      def put(to_url, params = {})
+        
+        file = params[:file]
+        headers = params[:headers] || {}
+        chunker = nil
+        rate_limit = nil
+        rate_limit = Cifrado::RateLimit.new(params[:bwlimit]) if params[:bwlimit]
+
+        if file
+          headers.merge!({ 'Content-Length' => File.size(file.path).to_s })
+
+          chunker = lambda do
+            chunk = file.read(4096).to_s
+
+            rate_limit.limit(chunk.size) if rate_limit
+
+            if block_given? and chunk.size > 0
+              yield chunk.size
+            end
+            chunk
+          end
+        end
+
+        Excon.put to_url,
+                  :headers => headers, 
+                  :request_block => chunker
+      end
+
+    end
+  end
+
+end
+

data/lib/cifrado/swift_client.rb

@@ -0,0 +1,313 @@
+require 'cifrado/streaming_uploader'
+require 'cifrado/streaming_downloader'
+
+module Cifrado 
+  
+  class SwiftClient
+
+    include Cifrado::Utils
+    attr_reader :api_key
+    attr_reader :username
+    attr_reader :auth_url
+
+    def initialize(auth_data = {})
+      @username = auth_data[:username]
+      @api_key  = auth_data[:api_key]
+      @auth_url = auth_data[:auth_url]
+      @tenant   = auth_data[:tenant]
+      @region   = auth_data[:region]
+      @connection_options = auth_data[:connection_options] || {}
+      @service_type = auth_data[:service_type] || 'object-store'
+      @endpoint_type = auth_data[:endpoint_type] || 'publicURL'
+      @password_salt = auth_data[:password_salt]
+
+      @connection_options.each do |k, v|
+        Excon.defaults[k] = v
+      end
+    end
+
+    def test_connection
+      authenticate_v2
+    end
+
+    def head(container = nil, object = nil)
+      begin 
+        if container and object
+          service.head_object(container, object).headers
+        elsif container
+          service.head_container(container).headers
+        else
+          (service.request :method => 'HEAD').headers
+        end
+      rescue Fog::Storage::OpenStack::NotFound
+        nil
+      end
+    end
+
+    def file_available?(container, object)
+      !head(container, object).nil?
+    end
+
+    def service
+      @service ||= authenticate_v2
+    end
+
+    def set_acl(acl, container, object = nil)
+      if object
+        raise NotImplementedError.new
+      else
+        service.request :path => Fog::OpenStack.escape(container), 
+                        :headers => { 'X-Container-Read' => acl },
+                        :expects => [201, 202],
+                        :method  => 'PUT'
+      end
+    end
+
+    def create_directory(container, wait_for_it = false)
+      create_container container, wait_for_it
+    end
+
+    def encrypted_upload(container, object, options = {})
+      cipher = CryptoEngineAES.new @api_key
+      encrypted_name = cipher.encrypt object
+      options[:headers] ||= {}
+      options[:headers]['X-Object-Meta-Encrypted-Name'] = encrypted_name
+      upload container, object, options
+    end
+
+    def upload(container, object, options = {})
+
+      raise ArgumentError.new("Invalid container") if container.nil?
+      raise ArgumentError.new("Invalid object") if object.nil?
+
+      # if :source_file present, object may be the destination
+      # path of the object and may not be the path to the
+      # real file
+      object_path = clean_object_name(options[:object_path] || object)
+      object_size = File.size(object)
+
+      Log.debug "Object size: #{humanize_bytes(object_size)}"
+
+      path = File.join('/', container, object_path)
+
+      Log.debug "X-Storage-Url: #{@storage_url}"
+
+      create_container container, true
+
+      Log.debug "Destination URI: " + @storage_url + path
+
+      pcallback = options[:progress_callback]
+      nchunk = 0
+      headers = { 'X-Auth-Token' => @auth_token }
+      if mime = mime_type(object)
+        headers['Content-Type'] = mime
+      end
+      if options[:headers]
+        headers = headers.merge options[:headers] 
+      end
+      if pcallback
+        res = Cifrado::StreamingUploader.put(
+            @storage_url + Fog::OpenStack.escape(path),
+            :headers => headers, 
+            :file => File.open(object),
+            :ssl_verify_peer => @connection_options[:ssl_verify_peer],
+            :bwlimit => options[:bwlimit]
+        ) { |bytes| pcallback.call(object_size, bytes) }
+      else
+        res = Cifrado::StreamingUploader.put(
+            @storage_url + Fog::OpenStack.escape(path),
+            :headers => headers, 
+            :file => File.open(object),
+            :ssl_verify_peer => @connection_options[:ssl_verify_peer],
+            :bwlimit => options[:bwlimit]
+        ) 
+      end
+
+      Log.debug "Upload response #{res.class}"
+      res
+    end
+
+    def user_agent
+      "Cifrado #{Cifrado::VERSION}"
+    end
+    
+    def stream(container, object, options = {})
+      raise ArgumentError.new "Invalid object" unless object
+
+      storage_url = service.credentials[:server_management_url]
+      storage_url << "/" unless storage_url =~ /\/$/
+
+      path = File.join(container, clean_object_name(object))
+
+      headers = {
+        "User-Agent" => "#{user_agent}",
+        "X-Auth-Token" => @auth_token
+      }
+      res = StreamingDownloader.get storage_url + Fog::OpenStack.escape(path),
+                                    nil,
+                                    :progress_callback => options[:progress_callback],
+                                    :connection_options => @connection_options,
+                                    :headers => headers,
+                                    :bwlimit => options[:bwlimit],
+                                    :stream => true
+    end
+
+    def download(container, object = nil, options = {})
+      if object
+        download_object container, object, options
+      else
+        dir = service.directories.get container
+        unless dir
+          Log.debug "Container #{container} not found"
+          raise "Container #{container} not found"
+        end
+        dest_dir = options[:output] || Dir.pwd
+        unless File.directory?(dest_dir)
+          raise ArgumentError.new "Directory #{dest_dir} does not exist"
+        end
+        dir.files.each do |f|
+          # Skip segments from segmented uploads
+          if f.key =~ /segments\/\d+\.\d{2}\/\d+/
+            Log.debug "Skipping segment #{f.key}"
+            next
+          end
+          unless options[:decrypt] == true
+            options[:output] = File.join(dest_dir, f.key)
+          end
+          download_object container, f.key, options
+        end
+        Excon::Response.new :status => 200
+      end
+    end
+    
+    #
+    # @returns [Integer]
+    #   * 0 if not available
+    #   * 1 if available
+    #   * 2 if available but MD5 does not match
+    def match(source_file, container, object_path = nil)
+      obj = object_path || source_file
+      headers = head container, clean_object_name(obj)
+      if headers
+        md5 = Digest::MD5.file(source_file).to_s
+        (md5 == headers['Etag']) ? 1 : 2
+      else
+        0
+      end
+    end
+
+    private
+    def download_object(container, object, options = {})
+      storage_url = service.credentials[:server_management_url]
+      storage_url << "/" unless storage_url =~ /\/$/
+      object = object[1..-1] if object =~ /^\//
+
+      raise ArgumentError.new "Invalid object" unless object
+      path = File.join(container, object)
+
+      dest_file = options[:output]
+      if dest_file 
+        if File.directory?(dest_file)
+          dest_file = File.join dest_file, object
+        end
+      else
+        Log.debug ":output option not specified, using current dir"
+        dest_file = File.join Dir.pwd, object
+      end
+
+      tmp_file = File.join Config.instance.cache_dir, "#{Time.now.to_f}.download"
+      Log.debug "Downloading tmp file to #{tmp_file}"
+
+      headers = {
+        "User-Agent" => "#{user_agent}",
+        "X-Auth-Token" => @auth_token
+      }
+      res = StreamingDownloader.get storage_url + Fog::OpenStack.escape(path),
+                                    tmp_file,
+                                    :progress_callback => options[:progress_callback],
+                                    :connection_options => @connection_options,
+                                    :headers => headers,
+                                    :bwlimit => options[:bwlimit]
+
+      #
+      # Try to decrypt the file if it was encrypted
+      #
+      if options[:decrypt]
+        encrypted_name = res['X-Object-Meta-Encrypted-Name']
+        if encrypted_name
+          Log.debug 'Encrypted filename found, decrypting'
+          if @password_salt
+            decrypted_name = decrypt_filename encrypted_name, @api_key + @password_salt
+          else
+            decrypted_name = decrypt_filename encrypted_name, @api_key
+          end
+          Log.debug "Decrypted filename: #{decrypted_name}"
+          if options[:output].nil?
+            dest_file = File.join(Dir.pwd, decrypted_name)
+          elsif File.directory?(options[:output])
+            dest_file = File.join(options[:output], decrypted_name)
+          else
+          end
+          Log.debug "Decrypted file output: #{dest_file}"
+          cs = CryptoServices.new :passphrase => options[:passphrase] 
+          tmp_file = cs.decrypt tmp_file, tmp_file + '.decrypted' 
+        else
+          Log.warn 'X-Object-Meta-Encrypted-Name header not found in object'
+          Log.warn 'Trying to decrypt anyways'
+        end
+      end
+
+      # if download is OK, move the file from cache to :output
+      # or to the current directory
+      if res.is_a? Net::HTTPOK
+        Log.debug "Moving download tmp file to #{dest_file}"
+        # Object name may have a path, create target directory 
+        # if not available
+        target_dir = File.dirname(dest_file)
+        unless File.directory?(target_dir)
+          Log.debug "Creating target directory #{target_dir}"
+          FileUtils.mkdir_p(target_dir)
+        end
+        FileUtils.mv tmp_file, dest_file
+      else
+        Log.debug "Download failed, deleting tmp file"
+        FileUtils.rm tmp_file
+      end
+      Excon::Response.new :body => res.body,
+                          :headers => res.to_hash,
+                          :status => res.code.to_i
+    end
+
+    def create_container(container, wait_for_it = false )
+      dir = service.directories.create :key => container
+      # Wait for the new container to be available
+      dir.wait_for { !service.directories.get(container).nil? } if wait_for_it
+      dir
+    end
+    
+    def add_manifest(container, object)
+      create_container container
+      service.put_object_manifest container, object
+    end
+    def authenticate_v2
+      return if @service
+      
+      Log.debug "Using keystone authentication"
+      @service = Fog::Storage.new :provider => 'OpenStack',
+                                  :openstack_auth_url => @auth_url,
+                                  :openstack_username => @username,
+                                  :openstack_api_key  => @api_key,
+                                  :openstack_tenant   => @tenant,
+                                  :openstack_service_type  => @service_type,
+                                  :openstack_endpoint_type => @endpoint_type,
+                                  :openstack_region => @region
+      @storage_url = @service.credentials[:server_management_url]
+      @auth_token  = @service.credentials[:token]
+      @service
+    end
+
+
+
+  end
+
+end