heirloom 0.6.1 → 0.7.0rc1

data/CHANGELOG

@@ -1,4 +1,14 @@
-## v0.6.1:
+## HEAD:
+
+* Adding catalog which will contain all heirloom regions & bases.
+* CLI will lookup base & region from catalog.
+* Removing base requirement for all subcmds outside download & setup.
+* Added teardown command to delete domains & buckets.
+* Enclosing simpledb domains in `s
+* Added catalog subcommand to list available Heirlooms from catalog.
+* Reducing log levels
+
+### v0.6.1:
 
 * Allowing engineer to specify region for metadata
 * Ensuring metadata is in an upload region

data/lib/heirloom/acl/s3.rb

@@ -28,7 +28,7 @@ module Heirloom
                                      :accounts => accounts
 
         accounts.each do |a|
-          logger.info "Authorizing #{a} to s3://#{bucket}/#{key}."
+          logger.debug "Authorizing #{a} to s3://#{bucket}/#{key}."
         end
         s3.put_object_acl bucket, key, grants
       end

data/lib/heirloom/archive/authorizer.rb

@@ -15,7 +15,7 @@ module Heirloom
 
       return false unless validate_format_of_accounts
 
-      @logger.info "Authorizing access to artifact."
+      @logger.info "Authorizing #{@accounts.join(', ')}."
 
       regions.each do |region|
         bucket = reader.get_bucket :region => region

data/lib/heirloom/archive/builder.rb

@@ -34,8 +34,6 @@ module Heirloom
 
       add_git_commit if args[:git]
 
-      @logger.info "Build complete."
-
       @local_build
     end
 
@@ -70,7 +68,7 @@ module Heirloom
                      'encrypted' => encrypted?,
                      'base'      => @base,
                      'id'        => @id }
-      @logger.info "Create artifact record #{@id}."
+      @logger.info "Adding entry #{@id}."
       sdb.put_attributes @domain, @id, attributes
     end
 

data/lib/heirloom/archive/destroyer.rb

@@ -12,9 +12,8 @@ module Heirloom
 
     def destroy(args)
       regions = args[:regions]
-      keep_domain = args[:keep_domain]
 
-      @logger.info "Destroying #{@name} - #{@id}"
+      @logger.info "Destroying #{@name} #{@id}"
 
       regions.each do |region|
         bucket = reader.get_bucket :region => region
@@ -22,7 +21,7 @@ module Heirloom
         key = "#{@id}.tar.gz"
 
         if bucket
-          @logger.info "Destroying 's3://#{bucket}/#{@name}/#{key}'."
+          @logger.debug "Destroying 's3://#{bucket}/#{@name}/#{key}'."
 
           s3_destroyer = Destroyer::S3.new :config => @config,
                                            :region => region
@@ -34,26 +33,10 @@ module Heirloom
       end
 
       sdb.delete @domain, @id
-
-      destroy_domain unless keep_domain
     end
 
     private
 
-    def destroy_domain
-
-      # Simple DB is eventually consisten
-      # Sleep for 3 sec for changes to reflect
-      Kernel.sleep 3
-
-      if sdb.domain_empty? @domain
-        @logger.info "Domain #{@domain} empty. Destroying."
-        sdb.delete_domain @domain
-      end
-
-      @logger.info "Destroy complete."
-    end
-
     def sdb
       @sdb ||= AWS::SimpleDB.new :config => @config
     end

data/lib/heirloom/archive/lister.rb

@@ -9,7 +9,7 @@ module Heirloom
     end
 
     def list(limit=10)
-      sdb.select("select * from #{@domain} where built_at > '2000-01-01T00:00:00.000Z'\
+      sdb.select("select * from `#{@domain}` where built_at > '2000-01-01T00:00:00.000Z'\
                   order by built_at desc limit #{limit}").keys
     end
 

data/lib/heirloom/archive/reader.rb

@@ -57,7 +57,7 @@ module Heirloom
     end
 
     def show
-      query = sdb.select "select * from #{@domain} where itemName() = '#{@id}'"
+      query = sdb.select "select * from `#{@domain}` where itemName() = '#{@id}'"
       items = query[@id] ? query[@id] : {}
       Hash.new.tap do |hash|
         items.each_pair.map do |key,value|

data/lib/heirloom/archive/teardowner.rb

@@ -0,0 +1,48 @@
+module Heirloom
+
+  class Teardowner
+
+    def initialize(args)
+      @config = args[:config]
+      @name   = args[:name]
+      @domain = "heirloom_#{@name}"
+      @logger = @config.logger
+      @region = @config.metadata_region
+    end
+
+    def delete_buckets(args)
+      regions       = args[:regions]
+      bucket_prefix = args[:bucket_prefix]
+
+      regions.each do |region|
+        bucket = "#{bucket_prefix}-#{region}"
+
+        if verifier.bucket_exists? :region        => region,
+                                   :bucket_prefix => bucket_prefix
+          @logger.info "Destroying bucket #{bucket} in #{region}."
+          s3 = AWS::S3.new :config => @config,
+                           :region => region
+          s3.delete_bucket bucket
+        end
+      end
+    end
+
+    def delete_domain
+      if verifier.domain_exists?
+        @logger.info "Destroying domain #{@name} in #{@region}."
+        sdb.delete_domain @domain 
+      end
+    end
+
+    private
+
+    def verifier
+      @verifier ||= Verifier.new :config => @config,
+                                 :name   => @name
+    end
+
+    def sdb
+      @sdb ||= AWS::SimpleDB.new :config => @config
+    end
+  end
+end

data/lib/heirloom/archive/verifier.rb

@@ -36,7 +36,7 @@ module Heirloom
         @logger.debug "Bucket '#{bucket}' exists in '#{region}'."
         true
       else
-        @logger.info "Bucket '#{bucket}' does not exist in '#{region}'."
+        @logger.debug "Bucket '#{bucket}' does not exist in '#{region}'."
         false
       end
     end
@@ -48,7 +48,7 @@ module Heirloom
         @logger.debug "Domain '#{@name}' exists in '#{region}'."
         true
       else
-        @logger.info "Domain '#{@name}' does not exist in '#{region}'."
+        @logger.debug "Domain '#{@name}' does not exist in '#{region}'."
         false
       end
     end

data/lib/heirloom/archive.rb

@@ -5,6 +5,7 @@ require 'heirloom/archive/updater.rb'
 require 'heirloom/archive/uploader.rb'
 require 'heirloom/archive/downloader.rb'
 require 'heirloom/archive/setuper.rb'
+require 'heirloom/archive/teardowner.rb'
 require 'heirloom/archive/writer.rb'
 require 'heirloom/archive/authorizer.rb'
 require 'heirloom/archive/destroyer.rb'
@@ -41,6 +42,14 @@ module Heirloom
       setuper.setup args
     end
 
+    def delete_buckets(args)
+      teardowner.delete_buckets args
+    end
+
+    def delete_domain
+      teardowner.delete_domain
+    end
+
     def update(args)
       updater.update args
     end
@@ -61,9 +70,8 @@ module Heirloom
       verifier.domain_exists?
     end
 
-    def destroy(args)
-      destroyer.destroy :regions     => regions,
-                        :keep_domain => args[:keep_domain]
+    def destroy
+      destroyer.destroy :regions => regions
     end
 
     def show
@@ -132,6 +140,11 @@ module Heirloom
                                :name   => @name
     end
 
+    def teardowner
+      @teardowner ||= Teardowner.new :config => @config,
+                                     :name   => @name
+    end
+
     def verifier
       @verifier ||= Verifier.new :config => @config,
                                  :name   => @name

data/lib/heirloom/aws/s3.rb

@@ -8,10 +8,10 @@ module Heirloom
         @config = args[:config]
         @region = args[:region]
 
-        @s3 = Fog::Storage.new :provider                 => 'AWS',
-                               :aws_access_key_id        => @config.access_key,
-                               :aws_secret_access_key    => @config.secret_key,
-                               :region                   => @region
+        @s3 = Fog::Storage.new :provider              => 'AWS',
+                               :aws_access_key_id     => @config.access_key,
+                               :aws_secret_access_key => @config.secret_key,
+                               :region                => @region
       end
 
       def delete_object(bucket_name, object_name, options = {})
@@ -22,6 +22,10 @@ module Heirloom
         @s3.directories.get bucket
       end
 
+      def delete_bucket(bucket)
+        @s3.delete_bucket bucket
+      end
+
       def get_object(bucket_name, object_name)
         @s3.get_object(bucket_name, object_name).body
       end

data/lib/heirloom/aws/simpledb.rb

@@ -44,10 +44,15 @@ module Heirloom
       end
 
       def count(domain)
-        body = @sdb.select("SELECT count(*) FROM #{domain}").body
+        body = @sdb.select("SELECT count(*) FROM `#{domain}`").body
         body['Items']['Domain']['Count'].first.to_i
       end
 
+      def item_count(domain, item)
+        query = "SELECT count(*) FROM `#{domain}` WHERE itemName() = '#{item}'"
+        @sdb.select(query).body['Items']['Domain']['Count'].first.to_i
+      end
+
     end
   end
 

data/lib/heirloom/catalog/add.rb

@@ -0,0 +1,31 @@
+module Heirloom
+  class Catalog
+    class Add
+
+      def initialize(args)
+        @config  = args[:config]
+        @name    = args[:name]
+        @logger  = @config.logger
+      end
+
+      def add_to_catalog(args)
+        regions = args[:regions]
+        base    = args[:base]
+
+        @logger.info "Adding #{@name} to catalog."
+
+        sdb.put_attributes 'heirloom', 
+                           "heirloom_#{@name}", 
+                           { "regions" => regions, "base" => base }
+
+      end
+
+      private
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog/delete.rb

@@ -0,0 +1,37 @@
+module Heirloom
+  class Catalog
+    class Delete
+
+      def initialize(args)
+        @config = args[:config]
+        @name    = args[:name]
+        @logger = @config.logger
+      end
+
+      def delete_from_catalog
+        domain  = "heirloom_#{@name}"
+
+        return false unless catalog_domain_exists?
+
+        @logger.info "Deleting #{@name} from catalog."
+
+        sdb.delete 'heirloom', domain
+      end
+
+      private
+
+      def catalog_domain_exists?
+        verify.catalog_domain_exists?
+      end
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+      def verify
+        @verify ||= Catalog::Verify.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog/list.rb

@@ -0,0 +1,22 @@
+module Heirloom
+  class Catalog
+    class List
+
+      def initialize(args)
+        @config = args[:config]
+      end
+
+      def all
+        query  = "SELECT * FROM heirloom"
+        sdb.select query
+      end
+
+      private
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog/setup.rb

@@ -0,0 +1,30 @@
+module Heirloom
+  class Catalog
+    class Setup
+
+      def initialize(args)
+        @config = args[:config]
+        @logger = @config.logger
+        @region = @config.metadata_region
+      end
+
+      def create_catalog_domain
+        unless verify.catalog_domain_exists?
+          @logger.info "Creating catalog in #{@region}."
+          sdb.create_domain "heirloom"
+        end
+      end
+
+      private
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+      def verify
+        @verify ||= Catalog::Verify.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog/show.rb

@@ -0,0 +1,35 @@
+module Heirloom
+  class Catalog
+    class Show
+
+      def initialize(args)
+        @config = args[:config]
+        @name   = args[:name]
+      end
+
+      def regions
+        lookup :name => @name, :attribute => 'regions'
+      end
+
+      def base
+        lookup(:name => @name, :attribute => 'base').first
+      end
+
+      private
+
+      def lookup(args)
+        name      = args[:name]
+        attribute = args[:attribute]
+        domain    = "heirloom_#{name}"
+        query     = "select #{attribute} from heirloom where itemName() = '#{domain}'"
+        result    = sdb.select(query)
+        result[domain][attribute]
+      end
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog/verify.rb

@@ -0,0 +1,39 @@
+module Heirloom
+  class Catalog
+    class Verify
+
+      def initialize(args)
+        @config = args[:config]
+        @logger = @config.logger
+        @region = @config.metadata_region
+      end
+
+      def catalog_domain_exists?
+        if sdb.domain_exists? 'heirloom'
+          @logger.debug "Catalog exists in #{@region}."
+          true
+        else
+          @logger.debug "Catalog does not exist in #{@region}."
+          false
+        end
+      end
+
+      def entry_exists_in_catalog?(entry)
+        if sdb.item_count('heirloom', "heirloom_#{entry}").zero?
+          @logger.debug "#{entry} does not exist in catalog."
+          false
+        else
+          @logger.debug "#{entry} exists in catalog."
+          true
+        end
+      end
+
+      private
+
+      def sdb
+        @sdb ||= AWS::SimpleDB.new :config => @config
+      end
+
+    end
+  end
+end

data/lib/heirloom/catalog.rb

@@ -0,0 +1,65 @@
+require 'heirloom/catalog/add.rb'
+require 'heirloom/catalog/delete.rb'
+require 'heirloom/catalog/list.rb'
+require 'heirloom/catalog/setup.rb'
+require 'heirloom/catalog/show.rb'
+require 'heirloom/catalog/verify.rb'
+
+module Heirloom
+  class Catalog
+
+    def initialize(args)
+      @config  = args[:config]
+      @name    = args[:name]
+    end
+
+    def create_catalog_domain
+      setup.create_catalog_domain
+    end
+
+    def delete_from_catalog
+      delete.delete_from_catalog
+    end
+
+    def add_to_catalog(args)
+      add.add_to_catalog args
+    end
+
+    def regions
+      show.regions
+    end
+
+    def base
+      show.base
+    end
+
+    def all
+      list.all
+    end
+
+    private
+
+    def add
+      @add ||= Catalog::Add.new :config => @config,
+                                :name   => @name
+    end
+
+    def delete
+      @delete ||= Catalog::Delete.new :config => @config,
+                                      :name   => @name
+    end
+
+    def list
+      @list ||= Catalog::List.new :config => @config
+    end
+
+    def setup
+      @setup ||= Catalog::Setup.new :config => @config
+    end
+
+    def show
+      @show ||= Catalog::Show.new :config => @config,
+                                  :name   => @name
+    end
+  end
+end

data/lib/heirloom/cli/authorize.rb

@@ -37,24 +37,24 @@ module Heirloom
           version Heirloom::VERSION
           banner <<-EOS
 
-Authorize access from another AWS account to an archive.
+Authorize access from another AWS account to an Heirloom.
 
 Usage:
 
 heirloom authorize -n NAME -i ID -a AWS_ACCOUNT1 -a AWS_ACCOUNT2
 
-Note: This will replace all current authorizations with those specified and make the archive private.
+Note: This will replace all current authorizations with those specified and make the Heirloom private.
 
 EOS
           opt :accounts, "AWS Account(s) email to authorize. Can be specified multiple times.", :type  => :string,
                                                                                                 :multi => true
           opt :help, "Display Help"
-          opt :id, "ID of the archive to authorize.", :type => :string
+          opt :id, "ID of the Heirloom to authorize.", :type => :string
           opt :level, "Log level [debug|info|warn|error].", :type    => :string,
                                                             :default => 'info'
           opt :metadata_region, "AWS region to store Heirloom metadata.", :type    => :string,   
                                                                           :default => 'us-west-1'
-          opt :name, "Name of archive.", :type => :string
+          opt :name, "Name of Heirloom.", :type => :string
           opt :aws_access_key, "AWS Access Key ID", :type => :string, 
                                                     :short => :none
           opt :aws_secret_key, "AWS Secret Access Key", :type => :string, 

data/lib/heirloom/cli/catalog.rb

@@ -0,0 +1,62 @@
+module Heirloom
+  module CLI
+    class Catalog
+
+      include Heirloom::CLI::Shared
+
+      def initialize
+        @opts = read_options
+        @logger = HeirloomLogger.new :log_level => @opts[:level]
+        @config = load_config :logger => @logger,
+                              :opts   => @opts
+
+        ensure_valid_options :provided => @opts,
+                             :required => [],
+                             :config   => @config
+        ensure_valid_region :region => @opts[:metadata_region],
+                            :config => @config
+        @catalog = Heirloom::Catalog.new :config  => @config
+      end
+      
+      def list
+        if @opts[:details]
+          jj catalog_with_heirloom_prefix_removed
+        else
+          jj catalog_with_heirloom_prefix_removed.keys
+        end
+      end
+
+      private
+
+      def catalog_with_heirloom_prefix_removed
+        Hash[@catalog.all.map { |k, v| [k.sub(/heirloom_/, ''), v] }]
+      end
+
+      def read_options
+        Trollop::options do
+          version Heirloom::VERSION
+          banner <<-EOS
+
+Show catalog of Heirlooms.
+
+Usage:
+
+heirloom catalog
+
+EOS
+          opt :help, "Display Help"
+          opt :level, "Log level [debug|info|warn|error].", :type    => :string,
+                                                            :default => 'info'
+          opt :details, "Include details."
+          opt :metadata_region, "AWS region to store Heirloom metadata.", :type    => :string,   
+                                                                          :default => 'us-west-1'
+          opt :aws_access_key, "AWS Access Key ID", :type => :string, 
+                                                    :short => :none
+          opt :aws_secret_key, "AWS Secret Access Key", :type => :string, 
+                                                        :short => :none
+        end
+      end
+
+    end
+  end
+end

data/lib/heirloom/cli/destroy.rb

@@ -27,7 +27,7 @@ module Heirloom
       end
       
       def destroy
-        @archive.destroy :keep_domain => false
+        @archive.destroy
       end
 
       private
@@ -37,7 +37,7 @@ module Heirloom
           version Heirloom::VERSION
           banner <<-EOS
 
-Destroy an archive.
+Destroy an Heirloom.
 
 Usage:
 
@@ -45,12 +45,12 @@ heirloom destroy -n NAME -i ID
 
 EOS
           opt :help, "Display Help"
-          opt :id, "ID of the archive to display.", :type => :string
+          opt :id, "ID of the Heirloom to display.", :type => :string
           opt :level, "Log level [debug|info|warn|error].", :type    => :string,
                                                             :default => 'info'
           opt :metadata_region, "AWS region to store Heirloom metadata.", :type    => :string,   
                                                                           :default => 'us-west-1'
-          opt :name, "Name of archive.", :type => :string
+          opt :name, "Name of Heirloom.", :type => :string
           opt :aws_access_key, "AWS Access Key ID", :type => :string, 
                                                     :short => :none
           opt :aws_secret_key, "AWS Secret Access Key", :type => :string,