awscli 0.1.0 → 0.1.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ODAzMjhiNjZkOWNhMjE3NWM5NjIzYTY3NzAwMWIyNmYxYWRmYWVjZg==
+    NjZmMTc3NTYxZjgzYjg3NTg3NTBlOWQ1NDg3Mjk2ODE4NjEyNjIzZg==
   data.tar.gz: !binary |-
-    MDk0ZDMyNDhlMjJlOWIwMjMzYThmMGZlZmUxYzhhNDczNDc3N2MyMQ==
+    OGQ1YzUwODYxZjZkYmQzYzY0YmE4OTU1MTJlYzk1MGE5MzZiNzk1YQ==
 !binary "U0hBNTEy":
   metadata.gz: !binary |-
-    MTM0MjBiOGJlYzk0ZWY3ZDFiNjNjYzdhMGEzOTE2ZDlkMDRlMzhkZTNhOTg2
-    YmMyYTc1NDdmZDJiYzdlMDA1NzU2NTUxMjdmNDM2YzYwNDg0MjE2MzRiZTM1
-    ZWU2M2U0MDBmYWIyYjIzNTc0YmNlYmRlNDY2YTljODg1YjAzMzI=
+    ZTEwNDU0NWQ4ZjRmMTI0YmI4ZDU0OThmYTYzZjU0YzE4NWJkZjNiYzJmN2I3
+    NWQ5MmJjMmI3YjU2MzlmNGI4ZDYyYjFjY2FhZjVhOTFjYTk2OTMxMjA0Y2Qz
+    MWM3NzUxMTZkMGM3MzE5MDk3YjM1YWJiYjczOGEzZjUyYjljY2I=
   data.tar.gz: !binary |-
-    N2RiYTAzZmY3YjFhZThlOTdkZmY4ZTk5M2FiYzYyNDRkMTRjMzQxMWM2MmVm
-    ZmE5ZDgyNWNmMTQ2NDZkMGFhNTk0NTgxNGZmNzA1OGMwOTA2MGVhMjRiNGQy
-    NGQ3ZTAxMzA0NThjN2Q2ZDhhNjczZmJhYjg4MjA3ZDIwYzMzNTM=
+    ZTUxN2ZlOTcyMWQyNzUyMjY4Yjk1MTZiZTViZGFhMTU1ZGI4YTU5YzQ5YTBj
+    OGQ3NmZjYWZlZmI2MjNiNzRjZjY5ODQ0NmQwM2EzMWQyNzZlNWQ5YTUwN2E1
+    YTY4MTRmZjQwYjQzYzMzMWI0NzliOGVmZGRiMWI0MTU0MTAyY2M=

data/lib/awscli/cli/ec2/ami.rb

@@ -4,16 +4,19 @@ module AwsCli
       require 'awscli/cli/ec2'
       class Ami < Thor
 
-        desc "list", "List Images"
+        desc "list [OPTIONS]", "List Images"
         method_option :filter, :aliases => "-f", :type => :hash, :desc => "filter the images based on filters"
         method_option :amazon_owned, :aliases => "-a", :type => :boolean, :default => false, :desc => "lists amazon owned images"
         method_option :show_filters, :aliases => "-s", :type => :boolean, :default => false, :desc => "filters available"
+        method_option :self_owned, :aliases => "-o", :type => :boolean, :default => false, :desc => "list self owned images"
         def list
           create_ec2_object
           if options[:amazon_owned]
             @ec2.list_amazon
           elsif options[:show_filters]
             @ec2.show_filters
+          elsif options[:self_owned]
+            @ec2.list_self
           else
             @ec2.list options[:filter]
           end
@@ -29,6 +32,14 @@ module AwsCli
           @ec2.create_image_from_instance options
         end
 
+        desc "deregister", "Deregisters the specified AMI. Once deregistered, the AMI cannot be used to launch new instances"
+        method_option :image_id, :aliases => "-i", :type => :string, :required => true, :desc => "ID of the AMI to deregister"
+        def deregister
+          create_ec2_object
+          @ec2.deregister options[:image_id]
+        end
+
+
         private
 
         def create_ec2_object

data/lib/awscli/cli/ec2/ebs.rb

@@ -4,7 +4,7 @@ module AwsCli
       require 'awscli/cli/ec2'
       class Ebs < Thor
 
-        desc "list", "List ELastic Block Storages"
+        desc "list [OPTIONS]", "List ELastic Block Storages"
         method_option :snapshots, :aliases => "-s", :type => :boolean, :default => false, :desc => "list snapshots"
         def list
           create_ec2_object
@@ -48,18 +48,15 @@ module AwsCli
 
         desc "delete_detached", "Delete all the volumes that are not in use"
         def delete_detached
-          if agree("Are you sure want to delete all the all volumes that are not in use ?  ")
-            puts
-            create_ec2_object
-            @ec2.delete_detached
-          end
+          create_ec2_object
+          @ec2.delete_detached
         end
 
         desc "create_snapshot", "Create a snapshot from volume"
         method_option :volume_id, :aliases => "-v", :banner => "VID", :required => true, :type => :string, :desc => "volume to make a snapshot from"
         def create_snapshot
           create_ec2_object
-          @ec2.create_snapshot
+          @ec2.create_snapshot options
         end
 
         desc "copy_snapshot", "Copy a snapshot to a different region"
@@ -67,7 +64,7 @@ module AwsCli
         method_option :snapshot_id, :aliases => "-i", :banner => "ID", :required => true, :type => :string, :desc => "Id of the snapshot"
         def copy_snapshot
           create_ec2_object
-          @ec2.copy_snapshot
+          @ec2.copy_snapshot options
         end
 
         desc "delete_snapshot", "Delete SnapShot"

data/lib/awscli/cli/s3/directories.rb

@@ -11,52 +11,52 @@ module AwsCli
         end
 
         desc "create", "Create an new S3 bucket"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket to create(name should be globally unique)"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket to create(name should be globally unique)"
         method_option :public, :aliases => "-p", :type => :boolean, :default => false, :desc => "makes the bucket publicly availble"
         # method_option :x_amz_acl, :aliases => "-x", :type => :string, :desc => "Permissions, must be in ['private', 'public-read', 'public-read-write', 'authenticated-read']"
         def create
           create_s3_object
-          @s3.create options
+          @s3.create options[:bucket], options[:public]
         end
 
         desc "delete", "Delete existing S3 bucket"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket to delete"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket to delete"
         def delete
           create_s3_object
-          @s3.delete options[:key]
+          @s3.delete options[:bucket]
         end
 
         desc "delete_rec", "Delete the bucket and all its contents"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket to delete"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket to delete"
         def delete_rec
           create_s3_object
-          @s3.delete_rec options[:key]
+          @s3.delete_rec options[:bucket]
         end
 
         desc "set_acl", "Change access control list for an S3 bucket"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket to change acl"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket to change acl"
         method_option :acl, :aliases => "-a", :type => :string, :required => true, :desc => "Permissions, must be in ['private', 'public-read', 'public-read-write', 'authenticated-read']"
         def set_acl
           create_s3_object
-          @s3.set_acl options[:key], options[:acl]
+          @s3.set_acl options[:bucket], options[:acl]
         end
 
         desc "get_acl", "Get access control list for an S3 bucket"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket to get acl"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket to get acl"
         def get_acl
           create_s3_object
-          @s3.get_acl options[:key]
+          @s3.get_acl options[:bucket]
         end
 
         desc "get_logging_status", "Get logging status for an S3 bucket"
-        method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket"
+        method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket"
         def get_logging_status
           create_s3_object
-          @s3.get_logging_status options[:key]
+          @s3.get_logging_status options[:bucket]
         end
 
         # desc "set_logging_status", "Change logging status for an S3 bucket"
-        # method_option :key, :aliases => "-k", :type => :string, :required => true, :desc => "name of the bucket"
+        # method_option :bucket, :aliases => "-b", :type => :string, :required => true, :desc => "name of the bucket"
         # method_option :owner, :aliases => "-o", :type => :hash, :banner => "ID:NAME", :desc => "set id and displayname of the owner"
         # method_option :grantee, :aliases => "-g", :type => :hash, :banner => "NAME:ID|EMAIL|URI", :desc => "Grantee hash containing, <Display name of the grantee>: <ID of the grantee (or) Email of the grantee (or) Uri of the group to grant access>"
         # method_option :permission, :aliases => "-p", :type => :string, :desc => "Permission, in [FULL_CONTROL, WRITE, WRITE_ACP, READ, READ_ACP]"
@@ -69,7 +69,7 @@ module AwsCli
         #   logging_status['AccessControlList'] = acl if acl
         #   puts "Empty logging_status will disable logging" if logging_status.nil?
         #   puts "#{logging_status}"
-        #   @s3.set_logging_status options[:key], logging_status
+        #   @s3.set_logging_status options[:bucket], logging_status
         # end
 
         private

data/lib/awscli/cli/s3/files.rb

@@ -19,6 +19,31 @@ module AwsCli
           @s3.upload_file options[:bucket_name], options[:file_path]
         end
 
+        desc "put_rec", "put a directory recusively into a specified bucket using multiple threads"
+        method_option :bucket_name, :aliases => "-b", :required => true, :desc => "name of the bucket to upload the dir to"
+        method_option :dir_path, :aliases => "-p", :required => true, :desc => "path of the dir to upload"
+        method_option :dest_path, :aliases => "-d", :desc => "optionally specify destination directory path to create"
+        method_option :thread_count, :aliases => "-t", :type => :numeric, :default => 5, :desc => "number of threads to use to upload files"
+        method_option :public, :type => :boolean, :default => false, :desc => "set ACL of files to public"
+        def put_rec
+          create_s3_object
+          @s3.upload_file_rec options
+        end
+
+        desc "put_big", "uploads a larger file (> 100 MB) using multipart uploads"
+        long_desc <<-DESC
+        Takes in a larger file, split the file into chunks and uploads the parts using amazon multipart uploads and the parts are aggregated at the amazons end.
+        DESC
+        method_option :bucket_name, :aliases => "-b", :required => true, :desc => "name of the bucket to upload the parts to"
+        method_option :file_path, :aliases => "-p", :required => true, :desc => "path of the file to upload"
+        method_option :tmp_dir, :aliases => "-t", :default => "/tmp", :desc => "path to a temperory location where file will be split into chunks"
+        method_option :acl, :aliases => "-a", :default => "private", :desc => "ACL to apply, to the object that is created after completing multipart upload, valid options in private | public-read | public-read-write | authenticated-read | bucket-owner-read | bucket-owner-full-control"
+        method_option :dest_path, :aliases => "-d", :desc => "optionally specify destination directory path to create"
+        def put_big
+          create_s3_object
+          @s3.multipart_upload options
+        end
+
         desc "get", "get a file from a bucket"
         method_option :bucket_name, :aliases => "-b", :required => true, :desc => "name of the bucket to download the file from"
         method_option :file_name, :aliases => "-f", :required => true, :desc => "name of file to download"

data/lib/awscli/ec2.rb

@@ -381,7 +381,8 @@ module Awscli
         if filter.nil?
           @@conn.images.all.table([:architecture, :id, :is_public, :platform, :root_device_type, :state])
         else
-          @@conn.images.all(filter).table([:architecture, :id, :is_public, :platform, :root_device_type, :state])
+          data = @@conn.images.all(filter)
+          data.empty? ? puts("No AMI's found for provided filters") : data.table([:architecture, :id, :is_public, :platform, :root_device_type, :state])
         end
       end
 
@@ -422,6 +423,11 @@ module Awscli
         @@conn.images.all('owner-alias' => 'amazon').table([:architecture, :id, :is_public, :platform, :root_device_type, :state])
       end
 
+      def list_self
+        response = @@conn.describe_images({'Owner' => 'self'}).body['imagesSet']
+        Formatador.display_table(response, ['architecture', 'imageId', 'isPublic', 'name', 'imageState', 'rootDeviceType', 'imageType'])
+      end
+
       def create_image_from_instance options
         abort "Invalid Instace: #{options[:instance_id]}" unless @@conn.servers.get(options[:instance_id])
         @@conn.create_image(
@@ -433,6 +439,13 @@ module Awscli
         puts "Created image from instance: #{options[:instance_id]}"
       end
 
+      def deregister image_id
+        image = @@conn.images.get(image_id)
+        abort "Cannot find image with id: #{image_id}" unless image
+        @@conn.deregister_image(image_id)
+        say "De-registerd image: <%= color('#{image_id}', :green) %>"
+      end
+
     end # => AMI
 
     class Ebs
@@ -485,12 +498,14 @@ module Awscli
       def delete_detached
         vols  = @@conn.volumes.all('status' => 'available')
         unless vols.empty?
-          puts "Deleting all volumes which are not in use ..."
-          vols.each do |vol|
-            vol.destroy
+          if agree("Are you sure want to delete all the all volumes that are not in use ?  ")
+            puts "Deleting all volumes which are not in use ..."
+            vols.each do |vol|
+              vol.destroy
+            end
           end
         else
-          puts "No volumes found, that are not in use found"
+          puts "No volumes found, that are 'not in use'"
         end
       end
 
@@ -501,7 +516,7 @@ module Awscli
       end
 
       def copy_snapshot options
-        abort "Cannot find snapshot: #{options[:snapshot_id]}" unless @@conn.snapshots.get(options[:snapshot_id])
+        # abort "Cannot find snapshot: #{options[:snapshot_id]}" unless @@conn.snapshots.get(options[:snapshot_id])
         @@conn.copy_snapshot(options[:snapshot_id], options[:source_region])
         puts "Copied snapshot"
       end
@@ -510,7 +525,7 @@ module Awscli
         snap = @@conn.snapshots.get(options[:snapshot_id])
         abort "Cannot find snapshot: #{options[:snapshot_id]}" unless snap
         snap.destroy
-        puts "Destroyed snapshot"
+        puts "Deleted snapshot"
       end
     end # => EBS
 

data/lib/awscli/s3.rb

@@ -1,6 +1,9 @@
 module Awscli
   module S3
     require 'thread'
+    require 'digest/md5'
+    require 'base64'
+    require 'fileutils'
 
     class Files
       def initialize connection, options = {}
@@ -30,6 +33,140 @@ module Awscli
         puts "Uploaded file: #{file_name} to bucket: #{dir_name}"
       end
 
+      def upload_file_rec options
+        dir_name, dir_path, threads_count, is_public = options[:bucket_name], options[:dir_path], options[:thread_count], options[:public]
+        dest_path = options[:dest_path] if options[:dest_path]
+        #check if bucket exists
+        bucket = @@conn.directories.get(dir_name)
+        abort "cannot find bucket: #{dir_name}" unless bucket
+        #check if passed path is a dir
+        dir = File.expand_path(dir_path)
+        abort "dir_path must be a dir" unless File.directory?(dir)
+        #add trailing slash to detination dir if is not passed
+        if dest_path && !dest_path.end_with?('/')
+          dest_path = "#{dest_path}/"
+        end
+        #remove trailing / from dir_path
+        dir = dir.chop if dir.end_with?('/')
+        #initializations
+        total_size = 0
+        files = Queue.new
+        threads = Array.new
+        semaphore = Mutex.new
+        file_number = 0
+
+        Dir.glob("#{dir}/**/*").select { |f| !File.directory?(f) }.each do |file|
+          files << file
+          total_size += File.size(file)
+        end
+
+        total_files = files.size
+        puts "Starting Upload using #{threads_count} threads"
+        threads_count.times do |count|
+          threads << Thread.new do
+            # Thread.current[:name] = "upload files #{count}"
+            # puts "...started thread '#{Thread.current[:name]}'...\n"
+            while not files.empty?
+              semaphore.synchronize do
+                file_number += 1
+              end
+              file = files.pop
+              key = file.gsub(dir, '')[1..-1]
+              dest = "#{dest_path}#{key}"
+              puts "[#{file_number}/#{total_files}] Uploading #{key} to s3://#{dir_name}/#{dest}"
+              bucket.files.create(
+                  :key => dest,
+                  :body => File.open(file),
+                  :public => is_public
+                )
+            end
+          end
+        end
+        # Wait for the threads to finish.
+        threads.each do |t|
+          begin
+            t.join
+          rescue RuntimeError => e
+            puts "Failure on thread #{t[:name]}: #{e.message}"
+          end
+        end
+        puts "Uploaded #{total_files} (#{total_size / 1024} KB)"
+      end
+
+      def multipart_upload options
+        bucket_name, file_path, tmp_loc, acl = options[:bucket_name], options[:file_path], options[:tmp_dir], options[:acl]
+        dir = @@conn.directories.get(bucket_name)
+        abort "cannot find bucket: #{bucket_name}" unless dir
+        file = File.expand_path(file_path)
+        abort "Invalid file path: #{file_path}" unless File.exist?(file)
+        dest_path = options[:dest_path] if options[:dest_path]
+        if dest_path
+          if !dest_path.end_with?('/')
+            #add trailing slash to detination dir if is not passed
+            dest_path = "#{dest_path}/"
+          elsif
+            #remove leading slash from destination dir path if exists
+            dest_path = dest_path[1..-1]
+          end
+        end
+
+        remote_file = if dest_path
+                        "#{dest_path}#{File.basename(file_path)}"
+                      else
+                        "#{File.basename(file_path)}"
+                      end
+
+        #get the file and split it
+        tmp_dir = "#{tmp_loc}/#{File.basename(file_path)}"
+        FileUtils.mkdir_p(tmp_dir)
+        #split the file into chunks => use smaller chunk sizes to minimize memory
+        puts "Spliting the file into 10M chunks ..."
+        `split -a3 -b10m #{file} #{tmp_dir}/#{File.basename(file_path)}`
+        abort "Cannot perform split on the file" unless $?.to_i == 0
+
+        parts = Dir.glob("#{tmp_loc}/#{File.basename(file_path)}/*").sort
+
+        #initiate the mulitpart upload & store the returned upload_id
+        puts "Initializing multipart upload"
+        multi_part_up = @@conn.initiate_multipart_upload(
+          bucket_name,                  # name of the bucket to create object in
+          remote_file,                  # name of the object to create
+          { 'x-amz-acl' => acl }
+        )
+        upload_id = multi_part_up.body["UploadId"]
+        puts "Upload ID: #{upload_id}"
+
+        part_ids = []
+
+        parts.each_with_index do |part, position|
+          part_number = (position + 1).to_s
+          puts "Uploading #{part} ..."
+          File.open part do |part_file|
+            response = @@conn.upload_part(
+                bucket_name,
+                # file[1..-1],
+                remote_file,
+                upload_id,
+                part_number,
+                part_file
+              )
+            part_ids << response.headers['ETag']
+          end
+        end
+
+        puts "Completing multipart upload ..."
+        response = @@conn.complete_multipart_upload(
+          bucket_name,
+          # file[1..-1],
+          remote_file,
+          upload_id,
+          part_ids
+        )
+        puts "Cleaning tmp_dir ..."
+        FileUtils.rm_rf(tmp_dir)
+        puts "Successfully completed multipart upload"
+      end
+
       def download_file dir_name, file_name, path
         dir = @@conn.directories.get(dir_name)
         abort "cannot find bucket: #{dir_name}" unless dir
@@ -44,6 +181,7 @@ module Awscli
       end
 
       def delete_file dir_name, file_name
+        #TODO: Handle globs for deletions
         dir = @@conn.directories.get(dir_name)
         abort "cannot find bucket: #{dir_name}" unless dir
         remote_file = dir.files.get(file_name)
@@ -71,9 +209,12 @@ module Awscli
         @@conn.directories.table
       end
 
-      def create options
-        dir = @@conn.directories.create(options)
-        puts "Create bucket: #{dir.key}"
+      def create bucket_name, is_public
+        dir = @@conn.directories.create(
+          :key => bucket_name,
+          :public => is_public
+        )
+        puts "Created bucket: #{dir.key}"
       end
 
       def delete dir_name

data/lib/awscli/version.rb

@@ -1,3 +1,3 @@
 module Awscli
-  VERSION = '0.1.0'
+  VERSION = '0.1.1'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: awscli
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.1.1
 platform: ruby
 authors:
 - Ashrith
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-29 00:00:00.000000000 Z
+date: 2013-03-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake