cifrado 0.1

data/lib/cifrado/cli/set_acl.rb

@@ -0,0 +1,10 @@
+module Cifrado
+  class CLI
+    desc "set-acl CONTAINER", 'Set an ACL on containers and objects'
+    option :acl, :type => :string, :required => true
+    def set_acl(container, object = nil)
+      client = client_instance
+      client.set_acl options[:acl], container
+    end
+  end
+end

data/lib/cifrado/cli/setup.rb

@@ -0,0 +1,64 @@
+module Cifrado
+  class CLI
+    desc "setup", "Initial Cifrado configuration"
+    def setup
+      config_file = File.join(ENV['HOME'], '.config/cifrado/cifradorc')
+      unless File.directory?('.config/cifrado')
+        FileUtils.mkdir_p '.config/cifrado'
+      end
+      if File.exist?(config_file)
+        Log.warn "Config file #{set_color config_file, :bold} already exist."
+        Log.warn "IMPORTANT: Make sure you backup the current config"
+        Log.warn "before saving a new configuration."
+        unless yes? "Continue?"
+          return
+        end
+        config = YAML.load_file(config_file)
+      else
+        config = {}
+      end
+
+
+      puts "Running cifrado setup..."
+      puts "Please provide OpenStack/Rackspace credentials."
+      puts
+      puts "Cifrado can save this settings in #{config_file}"
+      puts "for later use."
+      puts "The settings (password included) are saved unencrypted."
+      puts
+      config[:username] = ask(set_color('Username:', :bold))
+      config[:tenant]   = ask(set_color('Tenant:', :bold))
+      system 'stty -echo'
+      config[:password] = ask(set_color 'Password:', :bold)
+      system 'stty echo'
+      puts
+      config[:auth_url] = ask(set_color 'Auth URL:', :bold)
+
+      if !config[:secure_random]
+        # shit happens
+        if RUBY_VERSION >= '1.9'
+          config[:secure_random] = SecureRandom.hex.encode('UTF-8')
+        else
+          config[:secure_random] = SecureRandom.hex
+        end
+      end
+
+      if yes? "Do you want to save these settings? (y/n) "
+        if File.exist?(config_file)
+          backup = "#{config_file}.bak.#{Time.now.to_i}"
+          FileUtils.cp config_file, backup
+          Log.info "Saving backup file to #{backup}."
+        end
+        File.open(config_file, 'w') do |f| 
+          f.puts config.to_yaml
+          f.chmod 0600
+        end
+        @settings_saved = true
+        Log.info "Saved!"
+      end
+
+      Log.debug "Setup done"
+      config
+    end
+  end
+end

data/lib/cifrado/cli/stat.rb

@@ -0,0 +1,41 @@
+module Cifrado
+  class CLI 
+    desc "stat [CONTAINER] [OBJECT]", "Displays information for the account, container, or object."
+    def stat(container = nil, object = nil)
+      client = client_instance
+      creds = client.service.credentials
+      mgmt_url = creds[:server_management_url]
+      
+      reject_headers = ['Accept-Ranges', 'X-Trans-Id']
+      unless container and object
+        reject_headers << 'Content-Length'
+      end
+      reject_headers << 'Content-Type' unless object
+
+      object = clean_object_name(object) if object
+      headers = client.head(container, object)
+      if headers
+        puts "Account:".ljust(30) + File.basename(URI.parse(mgmt_url).path)
+        headers.sort.each do |k, v| 
+          next if reject_headers.include?(k)
+          if k == 'X-Timestamp'
+            puts "#{(k + ":").ljust(30)}#{v} (#{unix_time(v)})" 
+          elsif k == 'X-Account-Bytes-Used' or k == 'Content-Length'
+            puts "#{(k + ":").ljust(30)}#{v} (#{humanize_bytes(v)})" 
+          elsif k == 'X-Object-Meta-Encrypted-Name'
+            puts "#{(k + ":").ljust(30)}#{v}"
+          else
+            puts "#{(k + ":").ljust(30)}#{v}" 
+          end
+        end
+      else
+        if object
+          raise "Object not found."
+        else
+          raise "Container not found."
+        end
+      end
+      headers
+    end
+  end
+end

data/lib/cifrado/cli/upload.rb

@@ -0,0 +1,269 @@
+module Cifrado
+  class CLI
+
+    desc "upload CONTAINER FILE1 [FILE2] ...",
+         "Upload files or directories"
+    option :encrypt,
+           :desc => 'Encrypt files when uploading'
+
+    option :segments, 
+           :type => :numeric, 
+           :desc => "Split the data into segments"
+
+    option :strip_path, 
+           :type => :boolean,
+           :desc => 'Strip path from file when uploading'
+
+    option :progressbar,
+           :default => :fancy,
+           :desc => 'Progressbar style'
+
+    option :bwlimit,
+           :type => :numeric,
+           :desc => 'Limit the bandwidth available for uploads'
+
+    option :force, 
+           :type => :boolean,
+           :desc => 'Overwrite files when uploading'
+    def upload(container, *args)
+
+      if args.empty?
+        help 'upload'
+        raise "No files specified"
+      end
+      
+      client = client_instance 
+
+      uploaded = []
+      args.each do |file|
+        unless file and File.exist?(file)
+          raise "File '#{file}' does not exist"
+        end
+        
+        if File.directory?(file)
+          files = Dir["#{file}/**/*"].reject { |f| File.directory?(f) }
+        else
+          files = [file]
+        end
+
+        files.each do |f|
+          begin
+            if options[:segments]
+              uploaded << split_and_upload(client, container, f)
+            else
+              headers = client.head container, clean_object_name(f)
+              if headers
+                if headers['Etag'] == Digest::MD5.file(f).to_s
+                  if options[:force]
+                    Log.warn "File #{f} already uploaded and MD5 matches."
+                    Log.warn "Since --force was used, uploading it again."
+                    uploaded << upload_single(client, container, f)
+                  else
+                    Log.warn "File #{f} already uploaded and MD5 matches, skipping."
+                  end
+                else
+                  Log.warn "File #{f} already uploaded, but it has changed."
+                  if options[:force]
+                    Log.warn "Overwriting it as requested (--force)."
+                    uploaded << upload_single(client, container, f)
+                  else
+                    Log.warn "Since --force was not used, skipping it."
+                  end
+                end
+              else
+                uploaded << upload_single(client, container, f)
+              end
+            end
+          rescue Errno::ENOENT => e
+            Log.error "Error uploading #{f}: " + e.message
+          end
+        end
+      end
+      uploaded.flatten
+    end
+    
+    private
+    def upload_single(client, container, object)
+      fsize = File.size(object)
+      fbasename = File.basename(object)
+      Log.info "Uploading #{object} (#{humanize_bytes(fsize)})"
+
+      pb = Progressbar.new 1, 1, :style => options[:progressbar]
+
+      config = Cifrado::Config.instance
+      object_path = object
+      object_path = File.basename(object) if options[:strip_path]
+      if cs = needs_encryption
+        encrypted_file = File.join(config.cache_dir, File.basename(object))
+        Log.debug "Writing encrypted file to #{encrypted_file}"
+        encrypted_output = cs.encrypt object, 
+                                      encrypted_file
+        encrypted_name = encrypt_filename object, secure_password
+        client.upload container, 
+                      encrypted_output, 
+                      :headers => { 
+                        'X-Object-Meta-Encrypted-Name' => encrypted_name
+                      },
+                      :object_path => File.basename(encrypted_output),
+                      :progress_callback => pb.block,
+                      :bwlimit => bwlimit
+        object_path = File.basename(encrypted_output)
+        File.delete encrypted_output 
+      else
+        client.upload container, 
+                      object,
+                      :object_path => object_path,
+                      :progress_callback => pb.block,
+                      :bwlimit => bwlimit
+      end
+      object_path
+    end
+
+    def needs_encryption
+      return nil unless options[:encrypt]
+
+      tokens = options[:encrypt].split(':')
+      etype = tokens.first
+      if etype == 'a'
+        recipient = tokens[1..-1].join(':')
+        CryptoServices.new :type => :asymmetric, 
+                                    :recipient => recipient, 
+                                    :encrypt_name => true
+      elsif etype == 's' or etype == 'symmetric'
+        if etype == 'symmetric'
+          Log.info "Password to encrypt the data required"
+          system 'stty -echo'
+          passphrase = ask("Enter passphrase:")
+          puts
+          passphrase2 = ask("Repeat passphrase:")
+          puts
+          if passphrase != passphrase2
+            raise 'Passphrase does not match'
+          end
+          system 'stty echo'
+        else
+          passphrase = tokens[1..-1].join(':')
+        end
+        unless passphrase
+          raise "Invalid symmetric encryption passprase"
+        end
+        CryptoServices.new :type => :symmetric, 
+                                    :passphrase => passphrase, 
+                                    :encrypt_name => true
+      else
+        raise "Invalid encryption type #{etype}."
+      end
+    end
+
+    def encrypt_if_required(file)
+      if cs = needs_encryption
+        Log.debug "Encrypting object #{file}"
+        cache_dir = Cifrado::Config.instance.cache_dir
+        encrypted_output = cs.encrypt file, 
+                                      File.join(cache_dir, File.basename(file))
+      else
+        file
+      end
+    end
+
+    # FIXME: needs refactoring
+    def split_and_upload(client, container, object)
+      fbasename = File.basename(object)
+
+      # Encrypts the file if required
+      out = encrypt_if_required(object)
+
+      splitter = FileSplitter.new out, options[:segments]
+
+      if options[:encrypt]
+        target_manifest = File.basename(out)
+      else
+        target_manifest = (options[:strip_path] ? \
+                              File.basename(object) : clean_object_name(object))
+      end
+
+      Log.info "Segmenting file, #{options[:segments]} segments..."
+      Log.info "Uploading #{fbasename} segments"
+
+      segments_uploaded = []
+      splitter.split do |n, segment|
+        segment_size = File.size segment
+        hsegment_size = humanize_bytes segment_size
+        Log.info "Uploading segment #{n}/#{options[:segments]} (#{hsegment_size})"
+
+        segment_number = "%08d" % n
+        if options[:encrypt]
+          suffix = splitter.chunk_suffix + segment_number
+          obj_path = File.basename(out) + suffix
+          Log.debug "Encrypted object path: #{obj_path}"
+          encrypted_name = encrypt_filename object + suffix,
+                                            secure_password
+          headers = { 
+            'X-Object-Meta-Encrypted-Name' => encrypted_name 
+          }
+        else
+          obj_path = object + splitter.chunk_suffix + segment_number
+          Log.debug "Unencrypted object path #{obj_path}"
+          if options[:strip_path]
+            obj_path = File.basename(obj_path) 
+            Log.debug "Stripping path from object: #{obj_path}"
+          end
+          Log.debug "Uploading segment #{obj_path} (#{segment_size} bytes)..."
+          headers = {}
+        end
+
+        case client.match(segment, container + "_segments", obj_path)
+        when 1
+          Log.warn 'Segment already uploaded, skipping.'
+          File.delete segment
+          next
+        when 2
+          Log.warn 'Segment already uploaded but looks different. Updating.'
+        end
+
+        pb = Progressbar.new options[:segments],
+                             n,
+                             :style => options[:progressbar]
+
+        client.upload container + "_segments", 
+                      segment,
+                      :headers => headers,
+                      :object_path => obj_path,
+                      :progress_callback => pb.block,
+                      :bwlimit => bwlimit
+
+        File.delete segment
+        segments_uploaded << obj_path
+      end
+      
+      if options[:encrypt]
+        Log.debug "Deleting temporal encrypted file #{out}"
+        File.delete out 
+      end
+      
+      if segments_uploaded.size == 0
+        Log.warn 'All the segments have been previously uploaded.'
+        Log.warn 'Skipping manifest creation.'
+        return segments_uploaded
+      end
+
+      # We need this for segmented uploads
+      Log.debug "Adding manifest path #{target_manifest}"
+      xom = "#{Fog::OpenStack.escape(container + '_segments')}/" +
+            "#{Fog::OpenStack.escape(target_manifest)}"
+      headers = { 'X-Object-Manifest' => xom }
+      if options[:encrypt]
+        encrypted_name = encrypt_filename object, secure_password
+        headers['X-Object-Meta-Encrypted-Name'] = encrypted_name
+      end
+      client.create_directory container
+      client.service.put_object_manifest container, 
+                                         target_manifest,
+                                         headers  
+      segments_uploaded.insert 0, target_manifest
+
+      segments_uploaded
+    end
+
+  end
+end

data/lib/cifrado/config.rb

@@ -0,0 +1,18 @@
+require 'singleton'
+
+module Cifrado
+  class Config
+    include Singleton
+    
+    def initialize
+      unless File.directory?(cache_dir)
+        Log.debug "Creating cache dir: #{cache_dir}"
+        FileUtils.mkdir_p(cache_dir) 
+      end
+    end
+
+    def cache_dir
+      File.join(ENV['HOME'], '.cache/cifrado')
+    end
+  end
+end

data/lib/cifrado/core_ext/ruby18_base64.rb

@@ -0,0 +1,15 @@
+# Adds urlsafe_encode64 and urlsafe_decode64 for Ruby 1.8.
+if RUBY_VERSION < '1.9'
+  module Base64
+    module_function
+
+    def urlsafe_encode64(str)
+      [str].pack("m0").tr("\n",'').tr("+/", "-_")
+    end
+
+    def urlsafe_decode64(str)
+      str.tr("-_", "+/").unpack("m0").first
+    end
+  end
+end
+

data/lib/cifrado/crypto_services.rb

@@ -0,0 +1,218 @@
+require 'digest/sha2'
+require 'base64'
+
+module Cifrado
+  class CryptoServices
+
+    def initialize(options = {})
+      @options = options
+      @gpg_binary = @options[:gpg_binary] || '/usr/bin/gpg'
+      @gpg_extra_args = @options[:gpg_extra_args] || []
+      @gpg_extra_args = @gpg_extra_args.concat %w(--batch --yes)
+      @encrypt_name = @options[:encrypt_name] || false
+    end
+
+    def encrypt(file, output)
+      unless file and File.exist?(file)
+        raise ArgumentError.new "Invalid file #{file}"
+      end
+
+      if output.nil?
+        raise ArgumentError.new "Invalid output file path"
+      end
+
+      # so we can use --use-embedded-filename to recover the original
+      # filename
+      Log.debug "Setting GPG filename #{file} (Base64 URL encoded)"
+      clean_path = Pathname.new(file).cleanpath.to_s
+      @gpg_extra_args << "--set-filename '#{Base64.urlsafe_encode64(clean_path)}'"
+
+      check_args
+      raise ArgumentError.new("#{@gpg_binary} not found") unless File.exist?(@gpg_binary)
+
+      Log.debug "Encrypting file #{file}..."
+      if @encrypt_name and output != '-'
+        Log.debug "Scrambling file name #{file}..."
+        dir = File.dirname(output)
+        #output = File.join dir, (Digest::SHA2.new << (output + SecureRandom.hex)).to_s
+        output = File.join dir, (Digest::SHA2.new << file).to_s
+      end
+      
+      if output != '-'
+        @gpg_extra_args << ["--output #{Shellwords.escape(output)}"]
+      end
+
+      if @options[:type] == :asymmetric 
+        asymmetric file, output
+      else @options[:type] == :symmetric
+        symmetric file, output
+      end
+    end
+
+    def self.encrypted?(file)
+      output = `/usr/bin/gpg --yes --batch --no-use-agent --list-packets #{file} 2>&1`
+      Log.debug output
+      if output.match(/AES256 encrypted|encrypted with\s.*\skey,\sID\s.*created/m).nil?
+        return false
+      end
+      true
+    end
+
+    def decrypt(file, output)
+      unless file and File.exist?(file)
+        raise ArgumentError.new "Invalid file #{file}"
+      end
+
+      if output.nil?
+        raise ArgumentError.new "Invalid output file path"
+      end
+
+      raise ArgumentError.new("#{@gpg_binary} not found") unless File.exist?(@gpg_binary)
+
+      Log.debug "Decrypting file #{file}..."
+      
+      if output != '-'
+        @gpg_extra_args << "--output #{Shellwords.escape(output)}"
+      end
+
+      if @options[:passphrase]
+        @gpg_extra_args << "--no-use-agent --passphrase #{@options[:passphrase]}"
+      end
+      
+      cmd = "#{@gpg_binary} #{@gpg_extra_args.join(' ')} --decrypt #{Shellwords.escape(file)}"
+      Log.debug "Decrypting with: #{cmd}"
+      out = `#{cmd} 2>&1`
+      
+      if $? != 0
+        raise "Failed to decrypt file #{file}\n#{out}"
+      else
+        if @options[:delete_source]
+          File.delete file 
+          Log.debug "Deleting encrypted file #{file}"
+        end
+      end
+
+      Log.debug out
+      output
+    end
+
+    private
+
+    def check_args
+      if @options[:type] == :asymmetric 
+        unless @options[:recipient]
+          raise ArgumentError.new('Missing encryption recipient')
+        end
+      elsif @options[:type] == :symmetric
+        unless @options[:passphrase]
+          raise ArgumentError.new('Missing encryption passphrase')
+        end
+      else
+        raise ArgumentError.new "Unknown encryption type #{@options[:type]}"
+      end
+    end
+
+    def symmetric(file, output)
+      pfile = "/tmp/#{SecureRandom.hex}"
+      File.open(pfile, 'w') { |f| f.puts @options[:passphrase]; f.sync }
+      cmd = "#{@gpg_binary} #{@gpg_extra_args.join(' ')} --no-use-agent --passphrase-file #{pfile} --cipher-algo aes256 --symmetric #{Shellwords.escape(file)}"
+      #File.delete(pfile)
+      Log.debug "Encrypting with: #{cmd}"
+      out = `#{cmd} 2>&1`
+      
+      if $? != 0
+        raise "Failed to encrypt file #{file}\n#{out}"
+      else
+        if @options[:delete_source]
+          File.delete file 
+          Log.debug "Deleting unencrypted chunk #{file}"
+        end
+      end
+
+      Log.debug out
+      output
+    end
+
+    def asymmetric(file, output)
+      recipient = @options[:recipient]
+      cmd = "#{@gpg_binary} #{@gpg_extra_args.join(' ')} --no-encrypt-to --no-default-recipient --recipient '#{recipient}' --encrypt #{Shellwords.escape(file)}"
+      Log.debug "Encrypting with: #{cmd}"
+      out = `#{cmd} 2>&1`
+      
+      if $? != 0
+        raise "Failed to encrypt file #{file}\n#{out}"
+      else
+        if @options[:delete_source]
+          File.delete file 
+          Log.debug "Deleting unencrypted chunk #{file}"
+        end
+      end
+
+      Log.debug out
+      output
+    end
+
+  end
+
+  #
+  # Shamelessly stolen from Gibberish, from Mark Percival
+  # so I don't have to depend on yet another gem. 
+  #
+  # See: https://github.com/mdp/gibberish
+  #
+  # Added a few small modifications:
+  #
+  # * Use Base64 urlsafe_encode/decode
+  # 
+  class CryptoEngineAES
+
+    attr_reader :password, :size, :cipher
+
+    # Initialize with the password
+    #
+    # @param [String] password
+    # @param [Integer] size
+    def initialize(password, size=256)
+      @password = password
+      @size = size
+      @cipher = OpenSSL::Cipher::Cipher.new("aes-#{size}-cbc")
+    end
+
+    def encrypt(data, opts={})
+      salt = generate_salt(opts[:salt])
+      setup_cipher(:encrypt, salt)
+      e = cipher.update(data) + cipher.final
+      e = "Salted__#{salt}#{e}" #OpenSSL compatible
+      opts[:binary] ? e : Base64.urlsafe_encode64(e)
+    end
+    alias :enc :encrypt
+    alias :e :encrypt
+
+    def decrypt(data, opts={})
+      data = Base64.urlsafe_decode64(data) unless opts[:binary]
+      salt = data[8..15]
+      data = data[16..-1]
+      setup_cipher(:decrypt, salt)
+      cipher.update(data) + cipher.final
+    end
+    alias :dec :decrypt
+    alias :d :decrypt
+
+    private
+
+    def generate_salt(supplied_salt)
+      if supplied_salt
+        return supplied_salt.to_s[0,8].ljust(8,'.')
+      end
+      s = ''
+      8.times {s << rand(255).chr}
+      s
+    end
+
+    def setup_cipher(method, salt)
+      cipher.send(method)
+      cipher.pkcs5_keyivgen(password, salt, 1)
+    end
+  end
+
+end