heirloom 0.3.1 → 0.4.0

data/CHANGELOG

@@ -1,3 +1,10 @@
+## v0.4.0:
+
+* Prefixing domain name with heirloom_
+* Adding CLI support for specific AWS creds
+* Added specs
+* Added support to delete domain when no records exist
+
 ## v0.3.1:
 
 * Switched from Minitar to shell

data/README.md

@@ -1,7 +1,7 @@
 Heirloom
 ========
 
-Heirloom is packages and distributes files to cloud storage services.  Heirloom tracks metadata about those deployments, both about the file locations, as well as arbitrary metadata which can be set by an engineer or build process.
+Heirloom packages and distributes files to cloud storage services for deployment.  Heirloom tracks metadata about those deployments, both about the file locations, as well as arbitrary metadata which can be set by an engineer or build process.
 
 Installation
 ------------

data/Rakefile

@@ -1 +1,7 @@
 require "bundler/gem_tasks"
+require 'rspec/core/rake_task'
+
+desc 'Run specs'
+RSpec::Core::RakeTask.new do |t|
+  t.rspec_opts = %w(--color)
+end

data/lib/heirloom/archive/builder.rb

@@ -5,77 +5,77 @@ module Heirloom
 
   class Builder
 
-    attr_accessor :config, :id, :local_build, :logger, :name, :source
+    attr_writer :local_build
 
     def initialize(args)
-      self.config = args[:config]
-      self.name = args[:name]
-      self.id = args[:id]
-      self.logger = config.logger
-      sdb.create_domain name
+      @config = args[:config]
+      @name = args[:name]
+      @domain = "heirloom_#{@name}"
+      @id = args[:id]
+      @logger = @config.logger
+      sdb.create_domain @domain
     end
 
     def build(args)
-      self.source = args[:directory] ||= '.'
+      @source = args[:directory] ||= '.'
 
-      directory = Directory.new :path      => source,
+      directory = Directory.new :path      => @source,
                                 :exclude   => args[:exclude],
-                                :config    => config
+                                :config    => @config
 
       return false unless directory.build_artifact_from_directory
 
-      self.local_build = directory.local_build
+      @local_build = directory.local_build
 
       create_artifact_record
 
       add_git_commit if args[:git]
 
-      logger.info "Build complete."
+      @logger.info "Build complete."
 
-      local_build
+      @local_build
     end
 
     def cleanup
-      logger.info "Cleaning up local build #{local_build}."
-      File.delete local_build
+      @logger.info "Cleaning up local build #{@local_build}."
+      File.delete @local_build
     end
 
     private
 
     def add_git_commit
-      git = GitDirectory.new(:path => source)
-      commit = git.commit id
+      git = GitDirectory.new(:path => @source)
+      commit = git.commit @id
       if commit
         add_git_commit_to_artifact_record commit
       else
-        logger.warn "Could not find Git sha: #{id}."
+        @logger.warn "Could not find Git sha: #{@id}."
       end
     end
 
     def add_git_commit_to_artifact_record(commit)
-      attributes = { 'sha'             => id,
+      attributes = { 'sha'             => @id,
                      'abbreviated_sha' => commit.id_abbrev,
                      'message'         => commit.message,
                      'author'          => commit.author.name }
 
-      logger.info "Git sha: #{id}"
-      logger.info "Git abbreviated_sha: #{commit.id_abbrev}"
-      logger.info "Git message: #{commit.message}"
-      logger.info "Git author: #{commit.author.name}"
+      attributes.each_pair do |k, v|
+        @logger.info "Git #{k}: #{v}"
+      end
 
-      sdb.put_attributes name, id, attributes
+      sdb.put_attributes @domain, @id, attributes
     end
 
     def create_artifact_record
-      attributes = { 'built_by'        => "#{user}@#{hostname}",
-                     'built_at'        => Time.now.utc.iso8601,
-                     'id'              => id }
-      logger.info "Create artifact record #{id}."
-      sdb.put_attributes name, id, attributes
+      attributes = { 'built_by' => "#{user}@#{hostname}",
+                     'built_at' => Time.now.utc.iso8601,
+                     'id'       => @id }
+      @logger.info "Create artifact record #{@id}."
+      sdb.put_attributes @domain, @id, attributes
     end
 
     def sdb
-      @sdb ||= AWS::SimpleDB.new :config => config
+      @sdb ||= AWS::SimpleDB.new :config => @config
     end
 
     def user

data/lib/heirloom/archive/destroyer.rb

@@ -2,38 +2,47 @@ module Heirloom
 
   class Destroyer
 
-    attr_accessor :config, :id, :logger, :name
-
     def initialize(args)
-      self.config = args[:config]
-      self.name = args[:name]
-      self.id = args[:id]
-      self.logger = config.logger
+      @config = args[:config]
+      @name = args[:name]
+      @domain = "heirloom_#{@name}"
+      @id = args[:id]
+      @logger = @config.logger
     end
 
     def destroy(args)
       regions = args[:regions]
 
-      logger.info "Destroying #{@name} - #{@id}"
+      @logger.info "Destroying #{@name} - #{@id}"
 
       regions.each do |region|
         bucket = reader.get_bucket :region => region
 
-        key = "#{id}.tar.gz"
+        key = "#{@id}.tar.gz"
 
         if bucket
-          logger.info "Destroying 's3://#{bucket}/#{name}/#{key}'."
+          @logger.info "Destroying 's3://#{bucket}/#{@name}/#{key}'."
 
-          s3_destroyer = Destroyer::S3.new :config => config,
+          s3_destroyer = Destroyer::S3.new :config => @config,
                                            :region => region
 
-          s3_destroyer.destroy_file :key_name => key,
-                                    :key_folder => name,
-                                    :bucket => bucket
+          s3_destroyer.destroy_file :key_name   => key,
+                                    :bucket     => bucket,
+                                    :key_folder => @name
         end
       end
-      sdb.delete name, id
-      logger.info "Destroy complete."
+
+      sdb.delete @domain, @id
+
+      # Simple DB is eventually consisten
+      # Sleep for 3 sec for changes to reflect
+      Kernel.sleep 3
+
+      if sdb.domain_empty? @domain
+        @logger.info "Domain #{@domain} empty. Destroying."
+        sdb.delete_domain @domain
+      end
+      @logger.info "Destroy complete."
     end
 
     private
@@ -43,9 +52,9 @@ module Heirloom
     end
 
     def reader
-      @reader ||= Reader.new :config => config,
-                             :name   => name,
-                             :id     => id
+      @reader ||= Reader.new :config => @config,
+                             :name   => @name,
+                             :id     => @id
     end
 
   end

data/lib/heirloom/archive/downloader.rb

@@ -2,47 +2,45 @@ module Heirloom
 
   class Downloader
 
-    attr_accessor :config, :id, :name, :logger
-
     def initialize(args)
-      self.config = args[:config]
-      self.name = args[:name]
-      self.id = args[:id]
-      self.logger = config.logger
+      @config = args[:config]
+      @name = args[:name]
+      @id = args[:id]
+      @logger = @config.logger
     end
 
     def download(args)
       region = args[:region]
 
-      s3_downloader = Downloader::S3.new :config => config,
-                                         :logger => logger,
+      s3_downloader = Downloader::S3.new :config => @config,
+                                         :logger => @logger,
                                          :region => region
 
       bucket = reader.get_bucket :region => region
       key = reader.get_key :region => region
 
-      logger.info "Downloading s3://#{bucket}/#{key} from #{region}."
+      @logger.info "Downloading s3://#{bucket}/#{key} from #{region}."
 
       file = s3_downloader.download_file :bucket => bucket,
                                          :key    => key
 
       output = args[:output] ||= "./#{key.split('/').last}"
 
-      logger.info "Writing file to #{output}."
+      @logger.info "Writing file to #{output}."
 
       File.open(output, 'w') do |local_file|
         local_file.write file
       end
 
-      logger.info "Download complete."
+      @logger.info "Download complete."
     end
 
     private
 
     def reader
-      @reader ||= Reader.new :config => config,
-                             :name   => name,
-                             :id     => id
+      @reader ||= Reader.new :config => @config,
+                             :name   => @name,
+                             :id     => @id
     end
 
   end

data/lib/heirloom/archive/lister.rb

@@ -5,10 +5,11 @@ module Heirloom
     def initialize(args)
       @config = args[:config]
       @name = args[:name]
+      @domain = "heirloom_#{@name}"
     end
 
     def list(limit=10)
-      sdb.select("select * from #{@name} where built_at > '2000-01-01T00:00:00.000Z'\
+      sdb.select("select * from #{@domain} where built_at > '2000-01-01T00:00:00.000Z'\
                   order by built_at desc limit #{limit}").keys
     end
 

data/lib/heirloom/archive/reader.rb

@@ -2,21 +2,20 @@ module Heirloom
 
   class Reader
 
-    attr_accessor :config, :id, :name
-
     def initialize(args)
-      self.config = args[:config]
-      self.name = args[:name]
-      self.id = args[:id]
-      @logger = config.logger
+      @config = args[:config]
+      @name = args[:name]
+      @domain = "heirloom_#{@name}"
+      @id = args[:id]
+      @logger = @config.logger
     end
 
     def exists?
-      if show.any?
-        @logger.debug "Found entry for #{id} in SimpleDB."
+      if domain_exists? && show.any?
+        @logger.debug "Found entry for #{@id} in SimpleDB."
         true
       else
-        @logger.debug "Entry for #{id} not found in SimpleDB."
+        @logger.debug "Entry for #{@id} not found in SimpleDB."
         false
       end
     end
@@ -29,7 +28,7 @@ module Heirloom
     end
 
     def get_bucket(args)
-      @logger.debug "Looking for bucket in #{args[:region]} for #{id}"
+      @logger.debug "Looking for bucket in #{args[:region]} for #{@id}"
       url = get_url(args)
       if url
         bucket = url.gsub('s3://', '').split('/').first
@@ -53,9 +52,13 @@ module Heirloom
       end
     end
 
+    def count
+      sdb.count @domain
+    end
+
     def show
-      query = sdb.select "select * from #{name} where itemName() = '#{id}'"
-      items = query[id] ? query[id] : {}
+      query = sdb.select "select * from #{@domain} where itemName() = '#{@id}'"
+      items = query[@id] ? query[@id] : {}
       Hash.new.tap do |hash|
         items.each_pair.map do |key,value|
           hash[key] = value.first
@@ -65,15 +68,19 @@ module Heirloom
 
     private
 
+    def domain_exists?
+      sdb.domain_exists? @domain
+    end
+
     def get_url(args)
       return nil unless exists?
-      @logger.debug "Looking for #{args[:region]} endpoint for #{id}"
+      @logger.debug "Looking for #{args[:region]} endpoint for #{@id}"
       url = "#{args[:region]}-s3-url"
       if show[url]
-        @logger.debug "Found #{url} for #{id}."
+        @logger.debug "Found #{url} for #{@id}."
         show[url]
       else
-        @logger.debug "#{args[:region]} endpoint for #{id} not found."
+        @logger.debug "#{args[:region]} endpoint for #{@id} not found."
         nil
       end
     end

data/lib/heirloom/archive/updater.rb

@@ -7,6 +7,7 @@ module Heirloom
       @name = args[:name]
       @id = args[:id]
 
+      @domain = "heirloom_#{@name}"
       @logger = @config.logger
     end
 
@@ -14,7 +15,7 @@ module Heirloom
       attribute = args[:attribute]
       value = args[:value]
 
-      sdb.put_attributes @name, @id, { attribute => value }, { :replace => attribute }
+      sdb.put_attributes @domain, @id, { attribute => value }, { :replace => attribute }
       @logger.info "Updated #{@name} (#{@id}): #{attribute} = #{value}."
     end
 

data/lib/heirloom/archive.rb

@@ -13,8 +13,7 @@ module Heirloom
   class Archive
 
     def initialize(args)
-      @config = Config.new :config => args[:config],
-                           :logger => args[:logger]
+      @config = args[:config]
       @name = args[:name]
       @id = args[:id]
     end
@@ -28,6 +27,10 @@ module Heirloom
       builder.build args
     end
 
+    def count
+      reader.count
+    end
+
     def download(args)
       downloader.download args
     end

data/lib/heirloom/aws/simpledb.rb

@@ -6,9 +6,9 @@ module Heirloom
 
       def initialize(args)
         @config = args[:config]
-        @sdb = Fog::AWS::SimpleDB.new :aws_access_key_id => @config.access_key,
+        @sdb = Fog::AWS::SimpleDB.new :aws_access_key_id     => @config.access_key,
                                       :aws_secret_access_key => @config.secret_key,
-                                      :region => @config.primary_region
+                                      :region                => @config.primary_region
       end
 
       def domains
@@ -23,6 +23,14 @@ module Heirloom
         @sdb.create_domain(domain) unless domain_exists?(domain)
       end
 
+      def delete_domain(domain)
+        @sdb.delete_domain(domain)
+      end
+
+      def domain_empty?(domain)
+        count(domain).zero?
+      end
+
       def put_attributes(domain, key, attributes, options = {})
         @sdb.put_attributes domain, key, attributes, options
       end
@@ -35,6 +43,11 @@ module Heirloom
         @sdb.delete_attributes domain, key
       end
 
+      def count(domain)
+        body = @sdb.select("SELECT count(*) FROM #{domain}").body
+        body['Items']['Domain']['Count'].first.to_i
+      end
+
     end
   end
 

data/lib/heirloom/cli/authorize.rb

@@ -4,17 +4,20 @@ module Heirloom
   module CLI
     class Authorize
 
+      include Heirloom::CLI::Shared
+
       def initialize
         @opts = read_options
         @logger = HeirloomLogger.new :log_level => @opts[:level]
-
-        exit 1 unless CLI::Shared.valid_options? :provided => @opts,
-                                                 :required => [:accounts, 
-                                                               :name, :id],
-                                                 :logger   => @logger
+        @config = load_config :logger => @logger,
+                              :opts   => @opts
+        exit 1 unless valid_options? :provided => @opts,
+                                     :required => [:accounts, 
+                                                   :name, :id],
+                                     :logger   => @logger
         @archive = Archive.new :name   => @opts[:name],
                                :id     => @opts[:id],
-                               :logger => @logger
+                               :config => @config
       end
 
       def authorize
@@ -32,7 +35,7 @@ Authorize access from another AWS account to an archive.
 
 Usage:
 
-heirloom authorize -n NAME -i ID -a AWS_ACCOUNT1-a AWS_ACCOUNT2
+heirloom authorize -n NAME -i ID -a AWS_ACCOUNT1 -a AWS_ACCOUNT2
 
 Note: This will replace all current authorizations with those specified and make the archive private.
 
@@ -41,8 +44,10 @@ EOS
                                                                                                 :multi => true
           opt :help, "Display Help"
           opt :id, "id of the archive to authorize.", :type => :string
+          opt :key, "AWS Access Key", :type => :string
           opt :level, "Log level [debug|info|warn|error].", :type    => :string,
                                                             :default => 'info'
+          opt :secret, "AWS Secret Access Key", :type => :string
           opt :name, "Name of archive.", :type => :string
         end
       end

data/lib/heirloom/cli/build.rb

@@ -4,36 +4,42 @@ module Heirloom
   module CLI
     class Build
 
+      include Heirloom::CLI::Shared
+
       def initialize
         @opts = read_options
         @logger = HeirloomLogger.new :log_level => @opts[:level]
-        exit 1 unless CLI::Shared.valid_options? :provided => @opts, 
-                                                 :required => [:name, :id, :regions, 
-                                                               :bucket_prefix, 
-                                                               :directory],
-                                                 :logger   => @logger
+        @config = load_config :logger => @logger,
+                              :opts   => @opts
+
+        exit 1 unless valid_options? :provided => @opts, 
+                                     :required => [:name, :id, :region, 
+                                                   :base_prefix, 
+                                                   :directory],
+                                     :logger   => @logger
+
         @archive = Archive.new :name   => @opts[:name],
                                :id     => @opts[:id],
-                               :logger => @logger
+                               :config => @config
       end
 
       def build
-        unless @archive.buckets_exist? :bucket_prefix => @opts[:bucket_prefix],
-                                       :regions       => @opts[:regions]
+        unless @archive.buckets_exist? :bucket_prefix => @opts[:base_prefix],
+                                       :regions       => @opts[:region]
           @logger.error "Buckets do no exist in required regions."
           exit 1
         end
 
         @archive.destroy if @archive.exists?
                           
-        build = @archive.build :bucket_prefix   => @opts[:bucket_prefix],
-                               :directory       => @opts[:directory],
-                               :exclude         => @opts[:exclude].split(','),
-                               :git             => @opts[:git]
+        build = @archive.build :bucket_prefix => @opts[:base_prefix],
+                               :directory     => @opts[:directory],
+                               :exclude       => @opts[:exclude],
+                               :git           => @opts[:git]
 
         if build
-          @archive.upload :bucket_prefix   => @opts[:bucket_prefix],
-                          :regions         => @opts[:regions],
+          @archive.upload :bucket_prefix   => @opts[:base_prefix],
+                          :regions         => @opts[:region],
                           :public_readable => @opts[:public],
                           :file            => build
           @archive.cleanup
@@ -54,15 +60,17 @@ Build and upload a new archive.
 
 Usage:
 
-heirloom build -n NAME -i ID -b BUCKET_PREFIX -r REGION [-d DIRECTORY_TO_BUILD] [-p] [-g] [-e DIRECTORIES_TO_EXCLUDE] [-l LOG_LEVEL]
+heirloom build -n NAME -i ID -b BASE_PREFIX -r REGION1 -r REGION2 [-d DIRECTORY_TO_BUILD] [-p] [-g] [-e DIRECTORY_TO_EXCLUDE] [-l LOG_LEVEL]
 
 EOS
-          opt :bucket_prefix, "Bucket prefix which will be combined with region.\
+          opt :base_prefix, "Base bucket prefix which will be combined with region. \
 For example: -b 'test' -r 'us-west-1'  will expect bucket 'test-us-west-1' to be present", :type => :string
           opt :directory, "Source directory of build.", :type    => :string, 
                                                         :default => '.'
-          opt :exclude, "Comma spereate list of files or directories to exclude.", :type    => :string,
-                                                                                   :default => '.git'
+          opt :exclude, "File(s) or directorie(s) to exclude. \
+Can be specified multiple times.", :type  => :string, 
+                                   :multi => true
+          opt :key, "AWS Access Key ID", :type => :string
           opt :git, "Read git commit information from directory and set as archive attributes."
           opt :help, "Display Help"
           opt :id, "id for archive (when -g specified, assumed to be GIT sha).", :type => :string
@@ -70,8 +78,9 @@ For example: -b 'test' -r 'us-west-1'  will expect bucket 'test-us-west-1' to be
                                                             :default => 'info'
           opt :name, "Name of archive.", :type => :string
           opt :public, "Set this archive as public readable?"
-          opt :regions, "Region(s) to upload archive.  Can be set multiple times.", :type  => :string,
-                                                                                    :multi => true
+          opt :region, "Region(s) to upload archive.  Can be specified multiple times.", :type  => :string,
+                                                                                   :multi => true
+          opt :secret, "AWS Secret Access Key", :type => :string
         end
       end
 

data/lib/heirloom/cli/destroy.rb

@@ -2,17 +2,22 @@ module Heirloom
   module CLI
     class Destroy
 
+      include Heirloom::CLI::Shared
+
       def initialize
         @opts = read_options
         @logger = HeirloomLogger.new :log_level => @opts[:level]
-        exit 1 unless CLI::Shared.valid_options? :provided => @opts,
-                                                 :required => [:name, :id],
-                                                 :logger   => @logger
+        @config = load_config :logger => @logger,
+                              :opts   => @opts
+
+        exit 1 unless valid_options? :provided => @opts,
+                                     :required => [:name, :id],
+                                     :logger   => @logger
         @name = @opts[:name]
         @id = @opts[:id]
         @archive = Archive.new :name   => @name,
                                :id     => @id,
-                               :logger => @logger
+                               :config => @config
       end
       
       def destroy
@@ -35,9 +40,11 @@ heirloom destroy -n NAME -i ID [-l LOG_LEVEL]
 EOS
           opt :help, "Display Help"
           opt :id, "ID of the archive to display.", :type => :string
+          opt :key, "AWS Access Key ID", :type => :string
           opt :level, "Log level [debug|info|warn|error].", :type    => :string,
                                                             :default => 'info'
           opt :name, "Name of archive.", :type => :string
+          opt :secret, "AWS Secret Access Key", :type => :string
         end
       end
     end

data/lib/heirloom/cli/download.rb

@@ -2,16 +2,21 @@ module Heirloom
   module CLI
     class Download
 
+      include Heirloom::CLI::Shared
+
       def initialize
         @opts = read_options
         @logger = HeirloomLogger.new :log_level => @opts[:level]
-        exit 1 unless CLI::Shared.valid_options? :provided => @opts,
-                                                 :required => [:name, :id, :output],
-                                                 :logger   => @logger
+        @config = load_config :logger => @logger,
+                              :opts   => @opts
+
+        exit 1 unless valid_options? :provided => @opts,
+                                     :required => [:name, :id, :output],
+                                     :logger   => @logger
 
         @archive = Archive.new :name   => @opts[:name],
                                :id     => @opts[:id],
-                               :logger => @logger
+                               :config => @config
       end
       
       def download
@@ -35,12 +40,14 @@ heirloom download -n NAME -i ID -r REGION -o OUTPUT_FILE
 EOS
           opt :help, "Display Help"
           opt :id, "id of the archive to download.", :type => :string
+          opt :key, "AWS Access Key ID", :type => :string
           opt :name, "Name of archive.", :type => :string
           opt :level, "Log level [debug|info|warn|error].", :type    => :string,
                                                             :default => 'info'
           opt :output, "Location to download archive.", :type => :string
           opt :region, "Region to download archive.", :type    => :string,
                                                       :default => 'us-west-1'
+          opt :secret, "AWS Secret Access Key", :type => :string
         end
       end
     end