s3etag 0.0.1

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,30 @@
1
+ # S3etag
2
+
3
+ calculate AWS S3 etag by multi upload
4
+
5
+ # Install
6
+
7
+ % gem install s3etag
8
+
9
+ # Gem
10
+
11
+ require 's3etag'
12
+
13
+ p S3Etag.calc(:data => 'a' * 1000)
14
+ p S3Etag.calc(:data => 'a' * 1000, :threshold => 100, :min_part_size => 100)
15
+ p S3Etag.calc(:file => 'test.txt')
16
+ p S3Etag.calc(:file => '.text.text', :threshold => 100, :min_part_size => 100)
17
+
18
+ # Command line
19
+
20
+ % s3etag
21
+ s3etag file
22
+ -t, --threshold threshold
23
+ -p, --max-parts max-parts
24
+ -s, --min_part_size min_part_size
25
+
26
+ % s3etag text.txt
27
+ 91fdac689d4861c9ae7a0afa21a1f6b8-18
28
+
29
+ % s3etag -s 10000000 text.text
30
+ 32aafcd9748824e559b4dbd6b908f6fa-10
@@ -0,0 +1,15 @@
1
+ # require 'bundler/gem_tasks'
2
+
3
+ require 'rake/testtask'
4
+
5
+ Rake::TestTask.new("test") do |t|
6
+ t.libs << "test"
7
+ t.pattern = "test/**/*_test.rb"
8
+ # t.verbose = true
9
+ end
10
+
11
+ desc 'update gem'
12
+ task :update do
13
+ sh 'gem build s3etag.gemspec'
14
+ sh 'mv *.gem pkg'
15
+ end
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'optparse'
4
+ require 's3etag'
5
+
6
+ opt = {}
7
+ op = OptionParser.new
8
+ op.on('-t', '--threshold threshold'){ |v| opt[:threshold] = v.to_i }
9
+ op.on('-p', '--max-parts max-parts'){ |v| opt[:max_parts] = v.to_i }
10
+ op.on('-s', '--min_part_size min_part_size'){ |v| opt[:min_part_size] = v.to_i }
11
+
12
+ path = ARGV.last
13
+ if path && File.file?(path)
14
+ begin
15
+ op.parse ARGV
16
+ rescue OptionParser::InvalidOption => e
17
+ puts op.summarize(['invalid option.', 's3etag file'], 100)
18
+ else
19
+ puts S3Etag.calc(opt.merge(:file => path))
20
+ end
21
+ else
22
+ puts op.summarize(['s3etag file'], 100)
23
+ end
24
+
@@ -0,0 +1,44 @@
1
+ #!/usr/bin/env ruby
2
+
3
+ require 'digest/md5'
4
+ require 'optparse'
5
+ require 'stringio'
6
+
7
+ module S3Etag
8
+
9
+ def calc opt = {}
10
+ threshold = opt[:threshold] || 16 * 1024 * 1024
11
+ max_parts = opt[:max_parts] || 10000
12
+ min_part_size = opt[:min_part_size] || 5 * 1024 * 1024
13
+
14
+ if opt[:file]
15
+ total_size = File.stat(opt[:file]).size
16
+ io = open(opt[:file])
17
+ elsif opt[:data]
18
+ total_size = opt[:data].size
19
+ io = StringIO.new opt[:data]
20
+ else
21
+ raise ':file or :data is required'
22
+ end
23
+
24
+ r = nil
25
+ begin
26
+ if total_size <= threshold
27
+ r = Digest::MD5.hexdigest(io.read)
28
+ else
29
+ part_size = [(total_size.to_f / max_parts).ceil, min_part_size].max.to_i
30
+ md5s = []
31
+ while !io.eof?
32
+ md5s << Digest::MD5.digest(io.read(part_size))
33
+ end
34
+ r = Digest::MD5.hexdigest(md5s.join('')) + '-' + md5s.size.to_s
35
+ end
36
+ rescue Exception => e
37
+ raise e
38
+ ensure
39
+ io.close
40
+ end
41
+ end
42
+
43
+ module_function :calc
44
+ end
@@ -0,0 +1,16 @@
1
+ # -*- encoding: utf-8 -*-
2
+ $:.push File.expand_path("../lib", __FILE__)
3
+
4
+ Gem::Specification.new do |s|
5
+ s.name = "s3etag"
6
+ s.version = '0.0.1'
7
+ s.authors = ["swdyh"]
8
+ s.homepage = "https://github.com/swdyh/s3etag"
9
+ s.summary = %q{calculate AWS S3 etag by multi upload}
10
+ s.description = %q{calculate AWS S3 etag by multi upload}
11
+
12
+ s.files = `git ls-files`.split("\n")
13
+ s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
14
+ s.executables = ['s3etag']
15
+ s.require_paths = ["lib"]
16
+ end
@@ -0,0 +1,30 @@
1
+ require 'test/unit'
2
+ require 'digest/md5'
3
+ require 'fileutils'
4
+ require 'pathname'
5
+ require Pathname.new(__FILE__).realpath.join('..', '..', 'lib', 's3etag').to_s
6
+
7
+ class S3EtagTest < Test::Unit::TestCase
8
+
9
+ def test_calc
10
+ d1 = '0'
11
+ d2 = '0' * 1024 * 1024 * 16
12
+ d3 = '0' * (1024 * 1024 * 16 + 1)
13
+
14
+ tmp_dir = '_tmp'
15
+ FileUtils.mkdir_p tmp_dir
16
+ open(File.join(tmp_dir, 'd1.txt'), 'w') { |f| f.write(d1) }
17
+ open(File.join(tmp_dir,'d2.txt'), 'w') { |f| f.write(d2) }
18
+ open(File.join(tmp_dir,'d3.txt'), 'w') { |f| f.write(d3) }
19
+
20
+
21
+ assert_equal Digest::MD5.hexdigest(d1), S3Etag.calc(:data => d1)
22
+ assert_equal '50838bbf29aec4c6e62bee320d5c9138', S3Etag.calc(:data => d2)
23
+ assert_equal 'f41a8bb2924e4dccba846428d5ccd921-4', S3Etag.calc(:data => d3)
24
+ assert_equal Digest::MD5.hexdigest(d1), S3Etag.calc(:file => File.join(tmp_dir, 'd1.txt'))
25
+ assert_equal '50838bbf29aec4c6e62bee320d5c9138', S3Etag.calc(:file => File.join(tmp_dir, 'd2.txt'))
26
+ assert_equal 'f41a8bb2924e4dccba846428d5ccd921-4', S3Etag.calc(:file => File.join(tmp_dir, 'd3.txt'))
27
+
28
+ FileUtils.rm_rf tmp_dir
29
+ end
30
+ end
metadata ADDED
@@ -0,0 +1,72 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: s3etag
3
+ version: !ruby/object:Gem::Version
4
+ hash: 29
5
+ prerelease: false
6
+ segments:
7
+ - 0
8
+ - 0
9
+ - 1
10
+ version: 0.0.1
11
+ platform: ruby
12
+ authors:
13
+ - swdyh
14
+ autorequire:
15
+ bindir: bin
16
+ cert_chain: []
17
+
18
+ date: 2012-06-26 00:00:00 +09:00
19
+ default_executable:
20
+ dependencies: []
21
+
22
+ description: calculate AWS S3 etag by multi upload
23
+ email:
24
+ executables:
25
+ - s3etag
26
+ extensions: []
27
+
28
+ extra_rdoc_files: []
29
+
30
+ files:
31
+ - README.md
32
+ - Rakefile
33
+ - bin/s3etag
34
+ - lib/s3etag.rb
35
+ - s3etag.gemspec
36
+ - test/s3etag_test.rb
37
+ has_rdoc: true
38
+ homepage: https://github.com/swdyh/s3etag
39
+ licenses: []
40
+
41
+ post_install_message:
42
+ rdoc_options: []
43
+
44
+ require_paths:
45
+ - lib
46
+ required_ruby_version: !ruby/object:Gem::Requirement
47
+ none: false
48
+ requirements:
49
+ - - ">="
50
+ - !ruby/object:Gem::Version
51
+ hash: 3
52
+ segments:
53
+ - 0
54
+ version: "0"
55
+ required_rubygems_version: !ruby/object:Gem::Requirement
56
+ none: false
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ hash: 3
61
+ segments:
62
+ - 0
63
+ version: "0"
64
+ requirements: []
65
+
66
+ rubyforge_project:
67
+ rubygems_version: 1.3.7
68
+ signing_key:
69
+ specification_version: 3
70
+ summary: calculate AWS S3 etag by multi upload
71
+ test_files:
72
+ - test/s3etag_test.rb