s3cp 0.2.4 → 0.2.5

data/History.txt

@@ -1,5 +1,15 @@
+=== 0.2.6 / (Pending)
+
 === 0.2.5 / (Pending)
 
+* Added: "s3du" command to calculate disk usage.
+         Supports --depth, --regex, --unit parameters and more!
+
+* Changed: "s3ls -l" command now accepts --unit and --precision to configure
+           file size display.  Uses "smart" unit by default.
+
+* Changed: "s3ls -l" will now use S3CP_DATE_FORMAT environment if set.
+
 === 0.2.4 / 2012-02-27
 
 * Fixed: Handling of relative paths in s3cp.

data/bin/s3du

@@ -0,0 +1,3 @@
+#!/usr/bin/env ruby
+require 's3cp/s3du'
+

data/lib/s3cp/s3du.rb

@@ -0,0 +1,140 @@
+# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more contributor
+# license agreements.  See the NOTICE file  distributed with this work for
+# additional information regarding copyright ownership.  The ASF licenses this
+# file to you under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License.  You may obtain a copy of
+# the License at
+#
+#  http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'rubygems'
+require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
+require 'right_aws'
+require 'optparse'
+require 'date'
+require 'highline/import'
+
+require 's3cp/utils'
+
+# Parse arguments
+options = {}
+options[:precision] = 0
+options[:unit]      = nil
+options[:depth]     = 0
+options[:regex]     = nil
+
+op = OptionParser.new do |opts|
+  opts.banner = "s3ls [path]"
+  opts.separator ''
+
+  opts.on("--unit UNIT", "Force unit to use for file size display: #{S3CP::UNITS.join(', ')}.") do |unit|
+    options[:unit] = unit
+  end
+
+  opts.on("--precision PRECISION", "Precision used to display sizes, e.g. 3 => 0.123GB. (default 0)") do |precision|
+    options[:precision] = precision.to_i
+  end
+
+  opts.on("--depth DEPTH", "Depth to report space usage (default 0).") do |depth|
+    options[:depth] = depth.to_i
+  end
+
+  opts.on("--regex REGEX", "Regular expression to match keys.") do |regex|
+    options[:regex] = Regexp.new(regex)
+  end
+
+  opts.on_tail("-h", "--help", "Show this message") do
+    puts op
+    exit
+  end
+end
+op.parse!(ARGV)
+
+unless ARGV.size > 0
+  puts op
+  exit
+end
+
+url = ARGV[0]
+
+@options = options
+
+@bucket, @prefix = S3CP.bucket_and_key(url)
+fail "Your URL looks funny, doesn't it?" unless @bucket
+
+@s3 = S3CP.connect()
+
+s3_options = Hash.new
+s3_options[:prefix] = @prefix
+
+def depth(path)
+  path.count("/")
+end
+
+# Returns the index of the nth occurrence of substr if it exists, otherwise -1
+def nth_occurrence(str, substr, n)
+  pos = -1
+  if n > 0 && str.include?(substr)
+    i = 0
+    while i < n do
+      pos = str.index(substr, pos + substr.length) if pos != nil
+      i += 1
+    end
+  end
+  pos != nil && pos != -1 ? pos + 1 : -1
+end
+
+last_key     = nil
+actual_depth = depth(@prefix) + options[:depth] if options[:depth]
+total_size    = 0
+subtotal_size = 0
+
+def print(key, size)
+  size = S3CP.format_filesize(size, :unit => @options[:unit], :precision => @options[:precision])
+  puts ("%#{7 + @options[:precision]}s " % size) + key
+end
+
+@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
+
+  entries = page[:contents]
+  entries.each do |entry|
+    key  = entry[:key]
+    size = entry[:size]
+
+    if options[:regex].nil? || options[:regex].match(key)
+      current_key = if actual_depth
+        pos = nth_occurrence(key, "/", actual_depth)
+        pos ? key[0..pos-1] : prefix
+      end
+
+      if (last_key && last_key != current_key)
+        print(last_key, subtotal_size)
+        subtotal_size = size
+      else
+        subtotal_size += size
+      end
+
+      last_key = current_key
+      total_size += size
+    end
+  end
+end
+
+if last_key != nil
+  print(last_key, subtotal_size)
+end
+
+if options[:depth] > 0
+  print("", total_size)
+else
+  puts S3CP.format_filesize(total_size, :unit => options[:unit], :precision => options[:precision])
+end
+

data/lib/s3cp/s3ls.rb

@@ -26,8 +26,9 @@ require 's3cp/utils'
 
 # Parse arguments
 options = {}
-options[:date_format] = '%x %X'
+options[:date_format] = ENV['S3CP_DATE_FORMAT'] || '%x %X'
 options[:rows_per_page] = $terminal.output_rows if $stdout.isatty
+options[:precision] = 0
 
 op = OptionParser.new do |opts|
   opts.banner = "s3ls [path]"
@@ -57,6 +58,14 @@ op = OptionParser.new do |opts|
     options[:delimiter] = delimiter
   end
 
+  opts.on("--unit UNIT", "Force unit to use for file size display: #{S3CP::UNITS.join(', ')}.") do |unit|
+    options[:unit] = unit
+  end
+
+  opts.on("--precision PRECISION", "Precision used to display sizes, e.g. 3 => 0.123GB. (default 0)") do |precision|
+    options[:precision] = precision.to_i
+  end
+
   opts.on_tail("-h", "--help", "Show this message") do
     puts op
     exit
@@ -109,7 +118,9 @@ s3_options[:delimiter] = options[:delimiter] if options[:delimiter]
     if options[:long_format] && entry[:last_modified] && entry[:size]
       last_modified = DateTime.parse(entry[:last_modified])
       size = entry[:size]
-      puts "#{last_modified.strftime(options[:date_format])} #{ "%12d" % size} #{key}"
+      size = S3CP.format_filesize(size, :unit => options[:unit], :precision => options[:precision])
+      size = ("%#{7 + options[:precision]}s " % size)
+      puts "#{last_modified.strftime(options[:date_format])} #{size} #{key}"
     else
       puts key
     end

data/lib/s3cp/utils.rb

@@ -18,6 +18,9 @@
 module S3CP
   extend self
 
+  # Valid units for file size formatting
+  UNITS = %w{B KB MB GB TB EB ZB YB BB}
+
   # Connect to AWS S3
   def connect()
     access_key = ENV["AWS_ACCESS_KEY_ID"]     || raise("Missing environment variable AWS_ACCESS_KEY_ID")
@@ -41,5 +44,48 @@ module S3CP
     end
     [bucket, key]
   end
+
+  # Round a number at some `decimals` level of precision.
+  def round(n, decimals = 0)
+    (n * (10.0 ** decimals)).round * (10.0 ** (-decimals))
+  end
+
+  # Return a formatted string for a file size.
+  #
+  # Valid units are "b" (bytes), "kb" (kilobytes), "mb" (megabytes),
+  # "gb" (gigabytes), "tb" (terabytes), "eb" (exabytes), "zb" (zettabytes),
+  # "yb" (yottabytes), "bb" (brontobytes) and their uppercase equivalents.
+  #
+  # If :unit option isn't specified, the "best" unit is automatically picked.
+  # If :precision option isn't specified, the number is rounded to closest integer.
+  #
+  # e.g.  format_filesize( 512, :unit => "b",  :precision => 2) =>    "512B"
+  #       format_filesize( 512, :unit => "kb", :precision => 4) =>   "0.5KB"
+  #       format_filesize(1512, :unit => "kb", :precision => 3) => "1.477KB"
+  #
+  #       format_filesize(11789512) => "11MB"  # smart unit selection
+  #
+  #       format_filesize(11789512, :precision => 2) => "11.24MB"
+  #
+  def format_filesize(num, options = {})
+    precision = options[:precision] || 0
+    if options[:unit]
+      unit = options[:unit].upcase
+      fail "Invalid unit" unless UNITS.include?(unit)
+      num = num.to_f
+      for u in UNITS
+        if u == unit
+          s = "%0.#{precision}f" % round(num, precision)
+          return s + unit
+        end
+        num = num / 1024
+      end
+    else
+      e = (Math.log(num) / Math.log(1024)).floor
+      s = "%0.#{precision}f" % round((num.to_f / 1024**e), precision)
+      s + UNITS[e]
+    end
+  end
+
 end
 

data/lib/s3cp/version.rb

@@ -16,5 +16,5 @@
 # the License.
 
 module S3CP
-  VERSION = "0.2.4"
+  VERSION = "0.2.5"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: s3cp
 version: !ruby/object:Gem::Version 
-  hash: 31
+  hash: 29
   prerelease: 
   segments: 
   - 0
   - 2
-  - 4
-  version: 0.2.4
+  - 5
+  version: 0.2.5
 platform: ruby
 authors: 
 - Alex Boisvert
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-02-28 00:00:00 Z
+date: 2012-03-02 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   prerelease: false
@@ -136,6 +136,7 @@ executables:
 - s3cp
 - s3cp_complete
 - s3dir
+- s3du
 - s3ls
 - s3mod
 - s3rm
@@ -148,6 +149,7 @@ extra_rdoc_files:
 files: 
 - lib/s3cp/s3rm.rb
 - lib/s3cp/s3ls.rb
+- lib/s3cp/s3du.rb
 - lib/s3cp/version.rb
 - lib/s3cp/s3stat.rb
 - lib/s3cp/s3cp.rb
@@ -164,6 +166,7 @@ files:
 - bin/s3stat
 - bin/s3cat
 - bin/s3rm
+- bin/s3du
 - bin/s3dir
 homepage: 
 licenses: []