s3cp 0.2.4 → 0.2.5
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +10 -0
- data/bin/s3du +3 -0
- data/lib/s3cp/s3du.rb +140 -0
- data/lib/s3cp/s3ls.rb +13 -2
- data/lib/s3cp/utils.rb +46 -0
- data/lib/s3cp/version.rb +1 -1
- metadata +7 -4
data/History.txt
CHANGED
@@ -1,5 +1,15 @@
|
|
1
|
+
=== 0.2.6 / (Pending)
|
2
|
+
|
1
3
|
=== 0.2.5 / (Pending)
|
2
4
|
|
5
|
+
* Added: "s3du" command to calculate disk usage.
|
6
|
+
Supports --depth, --regex, --unit parameters and more!
|
7
|
+
|
8
|
+
* Changed: "s3ls -l" command now accepts --unit and --precision to configure
|
9
|
+
file size display. Uses "smart" unit by default.
|
10
|
+
|
11
|
+
* Changed: "s3ls -l" will now use S3CP_DATE_FORMAT environment if set.
|
12
|
+
|
3
13
|
=== 0.2.4 / 2012-02-27
|
4
14
|
|
5
15
|
* Fixed: Handling of relative paths in s3cp.
|
data/bin/s3du
ADDED
data/lib/s3cp/s3du.rb
ADDED
@@ -0,0 +1,140 @@
|
|
1
|
+
# Copyright (C) 2010-2012 Alex Boisvert and Bizo Inc. / All rights reserved.
|
2
|
+
#
|
3
|
+
# Licensed to the Apache Software Foundation (ASF) under one or more contributor
|
4
|
+
# license agreements. See the NOTICE file distributed with this work for
|
5
|
+
# additional information regarding copyright ownership. The ASF licenses this
|
6
|
+
# file to you under the Apache License, Version 2.0 (the "License"); you may not
|
7
|
+
# use this file except in compliance with the License. You may obtain a copy of
|
8
|
+
# the License at
|
9
|
+
#
|
10
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
#
|
12
|
+
# Unless required by applicable law or agreed to in writing, software
|
13
|
+
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
14
|
+
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
15
|
+
# License for the specific language governing permissions and limitations under
|
16
|
+
# the License.
|
17
|
+
|
18
|
+
require 'rubygems'
|
19
|
+
require 'extensions/kernel' if RUBY_VERSION =~ /1.8/
|
20
|
+
require 'right_aws'
|
21
|
+
require 'optparse'
|
22
|
+
require 'date'
|
23
|
+
require 'highline/import'
|
24
|
+
|
25
|
+
require 's3cp/utils'
|
26
|
+
|
27
|
+
# Parse arguments
|
28
|
+
options = {}
|
29
|
+
options[:precision] = 0
|
30
|
+
options[:unit] = nil
|
31
|
+
options[:depth] = 0
|
32
|
+
options[:regex] = nil
|
33
|
+
|
34
|
+
op = OptionParser.new do |opts|
|
35
|
+
opts.banner = "s3ls [path]"
|
36
|
+
opts.separator ''
|
37
|
+
|
38
|
+
opts.on("--unit UNIT", "Force unit to use for file size display: #{S3CP::UNITS.join(', ')}.") do |unit|
|
39
|
+
options[:unit] = unit
|
40
|
+
end
|
41
|
+
|
42
|
+
opts.on("--precision PRECISION", "Precision used to display sizes, e.g. 3 => 0.123GB. (default 0)") do |precision|
|
43
|
+
options[:precision] = precision.to_i
|
44
|
+
end
|
45
|
+
|
46
|
+
opts.on("--depth DEPTH", "Depth to report space usage (default 0).") do |depth|
|
47
|
+
options[:depth] = depth.to_i
|
48
|
+
end
|
49
|
+
|
50
|
+
opts.on("--regex REGEX", "Regular expression to match keys.") do |regex|
|
51
|
+
options[:regex] = Regexp.new(regex)
|
52
|
+
end
|
53
|
+
|
54
|
+
opts.on_tail("-h", "--help", "Show this message") do
|
55
|
+
puts op
|
56
|
+
exit
|
57
|
+
end
|
58
|
+
end
|
59
|
+
op.parse!(ARGV)
|
60
|
+
|
61
|
+
unless ARGV.size > 0
|
62
|
+
puts op
|
63
|
+
exit
|
64
|
+
end
|
65
|
+
|
66
|
+
url = ARGV[0]
|
67
|
+
|
68
|
+
@options = options
|
69
|
+
|
70
|
+
@bucket, @prefix = S3CP.bucket_and_key(url)
|
71
|
+
fail "Your URL looks funny, doesn't it?" unless @bucket
|
72
|
+
|
73
|
+
@s3 = S3CP.connect()
|
74
|
+
|
75
|
+
s3_options = Hash.new
|
76
|
+
s3_options[:prefix] = @prefix
|
77
|
+
|
78
|
+
def depth(path)
|
79
|
+
path.count("/")
|
80
|
+
end
|
81
|
+
|
82
|
+
# Returns the index of the nth occurrence of substr if it exists, otherwise -1
|
83
|
+
def nth_occurrence(str, substr, n)
|
84
|
+
pos = -1
|
85
|
+
if n > 0 && str.include?(substr)
|
86
|
+
i = 0
|
87
|
+
while i < n do
|
88
|
+
pos = str.index(substr, pos + substr.length) if pos != nil
|
89
|
+
i += 1
|
90
|
+
end
|
91
|
+
end
|
92
|
+
pos != nil && pos != -1 ? pos + 1 : -1
|
93
|
+
end
|
94
|
+
|
95
|
+
last_key = nil
|
96
|
+
actual_depth = depth(@prefix) + options[:depth] if options[:depth]
|
97
|
+
total_size = 0
|
98
|
+
subtotal_size = 0
|
99
|
+
|
100
|
+
def print(key, size)
|
101
|
+
size = S3CP.format_filesize(size, :unit => @options[:unit], :precision => @options[:precision])
|
102
|
+
puts ("%#{7 + @options[:precision]}s " % size) + key
|
103
|
+
end
|
104
|
+
|
105
|
+
@s3.interface.incrementally_list_bucket(@bucket, s3_options) do |page|
|
106
|
+
|
107
|
+
entries = page[:contents]
|
108
|
+
entries.each do |entry|
|
109
|
+
key = entry[:key]
|
110
|
+
size = entry[:size]
|
111
|
+
|
112
|
+
if options[:regex].nil? || options[:regex].match(key)
|
113
|
+
current_key = if actual_depth
|
114
|
+
pos = nth_occurrence(key, "/", actual_depth)
|
115
|
+
pos ? key[0..pos-1] : prefix
|
116
|
+
end
|
117
|
+
|
118
|
+
if (last_key && last_key != current_key)
|
119
|
+
print(last_key, subtotal_size)
|
120
|
+
subtotal_size = size
|
121
|
+
else
|
122
|
+
subtotal_size += size
|
123
|
+
end
|
124
|
+
|
125
|
+
last_key = current_key
|
126
|
+
total_size += size
|
127
|
+
end
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
if last_key != nil
|
132
|
+
print(last_key, subtotal_size)
|
133
|
+
end
|
134
|
+
|
135
|
+
if options[:depth] > 0
|
136
|
+
print("", total_size)
|
137
|
+
else
|
138
|
+
puts S3CP.format_filesize(total_size, :unit => options[:unit], :precision => options[:precision])
|
139
|
+
end
|
140
|
+
|
data/lib/s3cp/s3ls.rb
CHANGED
@@ -26,8 +26,9 @@ require 's3cp/utils'
|
|
26
26
|
|
27
27
|
# Parse arguments
|
28
28
|
options = {}
|
29
|
-
options[:date_format] = '%x %X'
|
29
|
+
options[:date_format] = ENV['S3CP_DATE_FORMAT'] || '%x %X'
|
30
30
|
options[:rows_per_page] = $terminal.output_rows if $stdout.isatty
|
31
|
+
options[:precision] = 0
|
31
32
|
|
32
33
|
op = OptionParser.new do |opts|
|
33
34
|
opts.banner = "s3ls [path]"
|
@@ -57,6 +58,14 @@ op = OptionParser.new do |opts|
|
|
57
58
|
options[:delimiter] = delimiter
|
58
59
|
end
|
59
60
|
|
61
|
+
opts.on("--unit UNIT", "Force unit to use for file size display: #{S3CP::UNITS.join(', ')}.") do |unit|
|
62
|
+
options[:unit] = unit
|
63
|
+
end
|
64
|
+
|
65
|
+
opts.on("--precision PRECISION", "Precision used to display sizes, e.g. 3 => 0.123GB. (default 0)") do |precision|
|
66
|
+
options[:precision] = precision.to_i
|
67
|
+
end
|
68
|
+
|
60
69
|
opts.on_tail("-h", "--help", "Show this message") do
|
61
70
|
puts op
|
62
71
|
exit
|
@@ -109,7 +118,9 @@ s3_options[:delimiter] = options[:delimiter] if options[:delimiter]
|
|
109
118
|
if options[:long_format] && entry[:last_modified] && entry[:size]
|
110
119
|
last_modified = DateTime.parse(entry[:last_modified])
|
111
120
|
size = entry[:size]
|
112
|
-
|
121
|
+
size = S3CP.format_filesize(size, :unit => options[:unit], :precision => options[:precision])
|
122
|
+
size = ("%#{7 + options[:precision]}s " % size)
|
123
|
+
puts "#{last_modified.strftime(options[:date_format])} #{size} #{key}"
|
113
124
|
else
|
114
125
|
puts key
|
115
126
|
end
|
data/lib/s3cp/utils.rb
CHANGED
@@ -18,6 +18,9 @@
|
|
18
18
|
module S3CP
|
19
19
|
extend self
|
20
20
|
|
21
|
+
# Valid units for file size formatting
|
22
|
+
UNITS = %w{B KB MB GB TB EB ZB YB BB}
|
23
|
+
|
21
24
|
# Connect to AWS S3
|
22
25
|
def connect()
|
23
26
|
access_key = ENV["AWS_ACCESS_KEY_ID"] || raise("Missing environment variable AWS_ACCESS_KEY_ID")
|
@@ -41,5 +44,48 @@ module S3CP
|
|
41
44
|
end
|
42
45
|
[bucket, key]
|
43
46
|
end
|
47
|
+
|
48
|
+
# Round a number at some `decimals` level of precision.
|
49
|
+
def round(n, decimals = 0)
|
50
|
+
(n * (10.0 ** decimals)).round * (10.0 ** (-decimals))
|
51
|
+
end
|
52
|
+
|
53
|
+
# Return a formatted string for a file size.
|
54
|
+
#
|
55
|
+
# Valid units are "b" (bytes), "kb" (kilobytes), "mb" (megabytes),
|
56
|
+
# "gb" (gigabytes), "tb" (terabytes), "eb" (exabytes), "zb" (zettabytes),
|
57
|
+
# "yb" (yottabytes), "bb" (brontobytes) and their uppercase equivalents.
|
58
|
+
#
|
59
|
+
# If :unit option isn't specified, the "best" unit is automatically picked.
|
60
|
+
# If :precision option isn't specified, the number is rounded to closest integer.
|
61
|
+
#
|
62
|
+
# e.g. format_filesize( 512, :unit => "b", :precision => 2) => "512B"
|
63
|
+
# format_filesize( 512, :unit => "kb", :precision => 4) => "0.5KB"
|
64
|
+
# format_filesize(1512, :unit => "kb", :precision => 3) => "1.477KB"
|
65
|
+
#
|
66
|
+
# format_filesize(11789512) => "11MB" # smart unit selection
|
67
|
+
#
|
68
|
+
# format_filesize(11789512, :precision => 2) => "11.24MB"
|
69
|
+
#
|
70
|
+
def format_filesize(num, options = {})
|
71
|
+
precision = options[:precision] || 0
|
72
|
+
if options[:unit]
|
73
|
+
unit = options[:unit].upcase
|
74
|
+
fail "Invalid unit" unless UNITS.include?(unit)
|
75
|
+
num = num.to_f
|
76
|
+
for u in UNITS
|
77
|
+
if u == unit
|
78
|
+
s = "%0.#{precision}f" % round(num, precision)
|
79
|
+
return s + unit
|
80
|
+
end
|
81
|
+
num = num / 1024
|
82
|
+
end
|
83
|
+
else
|
84
|
+
e = (Math.log(num) / Math.log(1024)).floor
|
85
|
+
s = "%0.#{precision}f" % round((num.to_f / 1024**e), precision)
|
86
|
+
s + UNITS[e]
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
44
90
|
end
|
45
91
|
|
data/lib/s3cp/version.rb
CHANGED
metadata
CHANGED
@@ -1,13 +1,13 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: s3cp
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
hash:
|
4
|
+
hash: 29
|
5
5
|
prerelease:
|
6
6
|
segments:
|
7
7
|
- 0
|
8
8
|
- 2
|
9
|
-
-
|
10
|
-
version: 0.2.
|
9
|
+
- 5
|
10
|
+
version: 0.2.5
|
11
11
|
platform: ruby
|
12
12
|
authors:
|
13
13
|
- Alex Boisvert
|
@@ -15,7 +15,7 @@ autorequire:
|
|
15
15
|
bindir: bin
|
16
16
|
cert_chain: []
|
17
17
|
|
18
|
-
date: 2012-02
|
18
|
+
date: 2012-03-02 00:00:00 Z
|
19
19
|
dependencies:
|
20
20
|
- !ruby/object:Gem::Dependency
|
21
21
|
prerelease: false
|
@@ -136,6 +136,7 @@ executables:
|
|
136
136
|
- s3cp
|
137
137
|
- s3cp_complete
|
138
138
|
- s3dir
|
139
|
+
- s3du
|
139
140
|
- s3ls
|
140
141
|
- s3mod
|
141
142
|
- s3rm
|
@@ -148,6 +149,7 @@ extra_rdoc_files:
|
|
148
149
|
files:
|
149
150
|
- lib/s3cp/s3rm.rb
|
150
151
|
- lib/s3cp/s3ls.rb
|
152
|
+
- lib/s3cp/s3du.rb
|
151
153
|
- lib/s3cp/version.rb
|
152
154
|
- lib/s3cp/s3stat.rb
|
153
155
|
- lib/s3cp/s3cp.rb
|
@@ -164,6 +166,7 @@ files:
|
|
164
166
|
- bin/s3stat
|
165
167
|
- bin/s3cat
|
166
168
|
- bin/s3rm
|
169
|
+
- bin/s3du
|
167
170
|
- bin/s3dir
|
168
171
|
homepage:
|
169
172
|
licenses: []
|