heirloom 0.5.0rc3 → 0.5.0rc4

data/CHANGELOG

@@ -2,7 +2,7 @@
 
 * Verify domain exists for all cmds except build & download
 * Removing requirement to check simpledb on download.
-* Adding requirement for base_prefix on download.
+* Adding requirement for base on download.
 * Fix error when .heirloom.yml does not exist.
 * Refactor cli option validation
 * Refactor cli specs
@@ -13,9 +13,16 @@
 * Verify directory given at build is a directory
 * Support archiving directories outside the cwd.
 * Fixed bug in git repo verification on build.
-* Added support for encryption
-* Removed short name and changed long name for AWS CLI Creds
-* Adding encrypted attribute by default
+* Added support for encryption.
+* Removed short name and changed long name for AWS CLI creds.
+* Adding encrypted attribute by default for all builds.
+* Change base_prefix CLI option to base
+* Changed subcommands build -> upload, update -> tag
+* Changed tag option updated_value -> value
+* On upload, Heirloom will attempt to create buckets which do not exist.
+* Will not destroy the sdb domain when cleanin up old versions on upload.
+* Hard coding gem versions
+* Validating email addresses in authorize
 
 ## v0.4.0:
 

data/README.md

@@ -1,7 +1,9 @@
 Heirloom
 ========
 
-Heirloom packages and distributes files to cloud storage services for deployment.  Heirloom tracks metadata about those deployments, both about the file locations, as well as arbitrary metadata which can be set by an engineer or build process.
+The goal of Heirloom is to securely and easily transport data to cloud hosted applciations.
+
+Heirloom creates archives from directories. Their archives are versioned and hosted in geographic distributed locations. Heirloom tracks metadata about those archives, both about the archive locations, as well as arbitrary tags which can be set by an engineer or process. It supports encryption and authorization to allow for securely transporting sensitive data over cloud storage services.
 
 Installation
 ------------

data/heirloom.gemspec

@@ -21,8 +21,7 @@ Gem::Specification.new do |s|
   # specify any dependencies here; for example:
   s.add_development_dependency "rspec"
 
-  s.add_runtime_dependency 'fog'
-  s.add_runtime_dependency 'grit'
-  s.add_runtime_dependency 'logger'
-  s.add_runtime_dependency 'trollop'
+  s.add_runtime_dependency 'fog', '~> 1.5.0'
+  s.add_runtime_dependency 'grit', '~> 2.5.0' 
+  s.add_runtime_dependency 'trollop', '= 2.0'
 end

data/lib/heirloom/archive.rb

@@ -4,6 +4,7 @@ require 'heirloom/archive/builder.rb'
 require 'heirloom/archive/updater.rb'
 require 'heirloom/archive/uploader.rb'
 require 'heirloom/archive/downloader.rb'
+require 'heirloom/archive/setuper.rb'
 require 'heirloom/archive/writer.rb'
 require 'heirloom/archive/authorizer.rb'
 require 'heirloom/archive/destroyer.rb'
@@ -15,8 +16,8 @@ module Heirloom
 
     def initialize(args)
       @config = args[:config]
-      @name = args[:name]
-      @id = args[:id]
+      @name   = args[:name]
+      @id     = args[:id]
     end
 
     def authorize(accounts)
@@ -36,6 +37,10 @@ module Heirloom
       downloader.download args
     end
 
+    def setup(args)
+      setuper.setup args
+    end
+
     def update(args)
       updater.update args
     end
@@ -56,8 +61,9 @@ module Heirloom
       verifier.domain_exists?
     end
 
-    def destroy
-      destroyer.destroy :regions => regions
+    def destroy(args)
+      destroyer.destroy :regions     => regions,
+                        :keep_domain => args[:keep_domain]
     end
 
     def show
@@ -121,6 +127,11 @@ module Heirloom
                                    :id     => @id
     end
 
+    def setuper
+      @setuper ||= Setuper.new :config => @config,
+                               :name   => @name
+    end
+
     def verifier
       @verifier ||= Verifier.new :config => @config,
                                  :name   => @name

data/lib/heirloom/archive/authorizer.rb

@@ -10,8 +10,10 @@ module Heirloom
     end
 
     def authorize(args)
+      @accounts = args[:accounts]
       regions = args[:regions]
-      accounts = args[:accounts]
+
+      return false unless validate_format_of_accounts
 
       @logger.info "Authorizing access to artifact."
 
@@ -23,15 +25,30 @@ module Heirloom
 
         s3_acl.allow_read_access_from_accounts :key_name   => @id,
                                                :key_folder => @name,
-                                               :bucket     => bucket,
-                                               :accounts   => accounts
+                                               :accounts   => @accounts,
+                                               :bucket     => bucket
       end
 
       @logger.info "Authorization complete."
+      true
     end
 
     private
 
+    def validate_format_of_accounts
+      @accounts.each do |account|
+        unless validate_email account
+          @logger.error "#{account} is not a valid email address."
+          return false
+        end
+      end
+    end
+
+    def validate_email email
+      email_pattern = (email =~ /^.*@.*\..*$/)
+      email_pattern.nil? ? false : true
+    end
+
     def reader
       @reader ||= Reader.new :config => @config,
                              :name   => @name,

data/lib/heirloom/archive/builder.rb

@@ -13,12 +13,12 @@ module Heirloom
       @domain = "heirloom_#{@name}"
       @id = args[:id]
       @logger = @config.logger
-      sdb.create_domain @domain
     end
 
     def build(args)
       @source = args[:directory] ||= '.'
       @secret = args[:secret]
+      @base = args[:base]
 
       directory = Directory.new :path      => @source,
                                 :exclude   => args[:exclude],
@@ -68,6 +68,7 @@ module Heirloom
       attributes = { 'built_by'  => "#{user}@#{hostname}",
                      'built_at'  => Time.now.utc.iso8601,
                      'encrypted' => encrypted?,
+                     'base'      => @base,
                      'id'        => @id }
       @logger.info "Create artifact record #{@id}."
       sdb.put_attributes @domain, @id, attributes

data/lib/heirloom/archive/destroyer.rb

@@ -12,6 +12,7 @@ module Heirloom
 
     def destroy(args)
       regions = args[:regions]
+      keep_domain = args[:keep_domain]
 
       @logger.info "Destroying #{@name} - #{@id}"
 
@@ -34,6 +35,13 @@ module Heirloom
 
       sdb.delete @domain, @id
 
+      destroy_domain unless keep_domain
+    end
+
+    private
+
+    def destroy_domain
+
       # Simple DB is eventually consisten
       # Sleep for 3 sec for changes to reflect
       Kernel.sleep 3
@@ -42,11 +50,10 @@ module Heirloom
         @logger.info "Domain #{@domain} empty. Destroying."
         sdb.delete_domain @domain
       end
+
       @logger.info "Destroy complete."
     end
 
-    private
-
     def sdb
       @sdb ||= AWS::SimpleDB.new :config => @config
     end

data/lib/heirloom/archive/setuper.rb

@@ -0,0 +1,48 @@
+module Heirloom
+
+  class Setuper
+
+    def initialize(args)
+      @config = args[:config]
+      @name   = args[:name]
+      @domain = "heirloom_#{@name}"
+      @logger = @config.logger
+    end
+
+    def setup(args)
+      @regions       = args[:regions]
+      @bucket_prefix = args[:bucket_prefix]
+      create_buckets
+      create_domain
+    end
+
+    private
+
+    def create_buckets
+      @regions.each do |region|
+        bucket = "#{@bucket_prefix}-#{region}"
+       
+        unless verifier.bucket_exists? :region        => region,
+                                       :bucket_prefix => @bucket_prefix
+          @logger.info "Creating bucket #{bucket} in #{region}."
+          s3 = AWS::S3.new :config => @config,
+                           :region => region
+          s3.put_bucket bucket, region
+        end
+      end
+    end
+
+    def create_domain
+      sdb.create_domain @domain unless verifier.domain_exists?
+    end
+
+    def verifier
+      @verifier ||= Verifier.new :config => @config,
+                                 :name   => @name
+    end
+
+    def sdb
+      @sdb ||= AWS::SimpleDB.new :config => @config
+    end
+  end
+end

data/lib/heirloom/archive/verifier.rb

@@ -14,23 +14,33 @@ module Heirloom
       result = true
 
       regions.each do |region|
-        bucket = "#{bucket_prefix}-#{region}"
-        
-        s3 ||= AWS::S3.new :config => @config,
-                           :region => region
-
-        if s3.get_bucket bucket
-          @logger.debug "#{bucket} exists in #{region}"
-        else
-          @logger.debug "#{bucket} in #{region} does not exist"
+        unless bucket_exists? :region        => region,
+                              :bucket_prefix => bucket_prefix
           result = false
         end
-
       end
 
       result
     end
 
+    def bucket_exists?(args)
+      bucket_prefix = args[:bucket_prefix]
+      region = args[:region]
+
+      bucket = "#{bucket_prefix}-#{region}"
+      
+      s3 ||= AWS::S3.new :config => @config,
+                         :region => region
+
+      if s3.get_bucket bucket
+        @logger.debug "Bucket #{bucket} exists in #{region}"
+        true
+      else
+        @logger.debug "Bucket #{bucket} in #{region} does not exist"
+        false
+      end
+    end
+
     def domain_exists?
       domain = "heirloom_#{@name}"
       sdb.domain_exists? domain

data/lib/heirloom/aws/s3.rb

@@ -34,6 +34,13 @@ module Heirloom
         @s3.put_object_acl(bucket, key, grants)
       end
 
+      def put_bucket(bucket_name, region)
+        region = nil if region == 'us-east-1'
+        options = { 'LocationConstraint' => region,
+                    'x-amz-acl'          => 'private' }
+        @s3.put_bucket bucket_name, options
+      end
+
     end
   end
 end

data/lib/heirloom/cipher/data.rb

@@ -13,17 +13,20 @@ module Heirloom
         data   = args[:data]
         secret = args[:secret]
 
-        return data unless secret
+        return data unless args[:secret]
 
         @logger.info "Secret provided. Decrypting archive."
 
         @aes = OpenSSL::Cipher::AES256.new(:CBC)
         @aes.decrypt
-        @aes.key = secret
+        @aes.key = Digest::SHA256.hexdigest secret
         @aes.iv = data.slice!(0,16)
         begin
           @aes.update(data) + @aes.final
         rescue OpenSSL::Cipher::CipherError => e
+          if e.message == 'wrong final block length'
+            @logger.error 'This archive does not appear to be encrypted.'
+          end
           @logger.error "Unable to decrypt archive: '#{e.message}'"
           false
         end

data/lib/heirloom/cipher/file.rb

@@ -18,7 +18,7 @@ module Heirloom
 
         @aes.encrypt
         @aes.iv = iv
-        @aes.key = secret
+        @aes.key = Digest::SHA256.hexdigest secret
 
         # Need to refactor to be less complex
         # Additionally tests to do fully cover logic

data/lib/heirloom/cli.rb

@@ -1,12 +1,12 @@
-require 'trollop'
 require 'json'
+require 'trollop'
 
 require 'heirloom/cli/shared'
 require 'heirloom/cli/authorize'
-require 'heirloom/cli/build'
+require 'heirloom/cli/upload'
 require 'heirloom/cli/list'
 require 'heirloom/cli/show'
-require 'heirloom/cli/update'
+require 'heirloom/cli/tag'
 require 'heirloom/cli/download'
 require 'heirloom/cli/destroy'
 
@@ -16,25 +16,25 @@ module Heirloom
       cmd = ARGV.shift
 
       case cmd
-      when 'list'
-        CLI::List.new.list
-      when 'show'
-        CLI::Show.new.show
-      when 'build'
-        CLI::Build.new.build
       when 'authorize'
         CLI::Authorize.new.authorize
-      when 'update'
-        CLI::Update.new.update
-      when 'download'
-        CLI::Download.new.download
       when 'destroy', 'delete'
         CLI::Destroy.new.destroy
+      when 'download'
+        CLI::Download.new.download
+      when 'list'
+        CLI::List.new.list
+      when 'show'
+        CLI::Show.new.show
+      when 'update', 'tag'
+        CLI::Tag.new.tag
+      when 'build', 'upload'
+        CLI::Upload.new.upload
       when '-v'
         puts Heirloom::VERSION
       else
         puts "Unkown command: '#{cmd}'." unless cmd == '-h'
-        puts "heirloom [list|show|build|authorize|update|download|destroy] OPTIONS"
+        puts "heirloom [list|show|upload|authorize|tag|download|destroy] OPTIONS"
         puts "Append -h for help on specific command."
       end
     end

data/lib/heirloom/cli/authorize.rb

@@ -22,7 +22,9 @@ module Heirloom
       end
 
       def authorize
-        @archive.authorize @opts[:accounts]
+        unless @archive.authorize @opts[:accounts]
+          exit 1
+        end
       end
 
       private

data/lib/heirloom/cli/destroy.rb

@@ -24,7 +24,7 @@ module Heirloom
       end
       
       def destroy
-        @archive.destroy
+        @archive.destroy :keep_domain => false
       end
 
       private

data/lib/heirloom/cli/shared.rb

@@ -15,8 +15,8 @@ module Heirloom
         config = args[:config]
         secret = args[:secret]
         logger = config.logger
-        if secret && secret.length != 32
-          logger.error "Secret must be exactly 32 characters long."
+        if secret && secret.length < 8
+          logger.error "Secret must be at least 8 characters long."
           exit 1
         end
       end
@@ -27,15 +27,14 @@ module Heirloom
         config   = args[:config]
         logger   = config.logger
 
-        required << :aws_key unless config.access_key
-        required << :aws_secret unless config.secret_key
+        required << :aws_access_key unless config.access_key
+        required << :aws_secret_key unless config.secret_key
 
-        missing_opts = required.map do |opt|
+        missing_opts = required.sort.map do |opt|
           case provided[opt]
-          when nil
-            "Option '#{opt} (-#{opt[0]})' required but not specified."
-          when []
-            "Option '#{opt} (-#{opt[0]})' required but not specified."
+          when nil, []
+            pretty_opt = opt.to_s.gsub('_', '-')
+            "Option '#{pretty_opt}' required but not specified."
           end
         end