s3etag 0.0.1

data/README.md

@@ -0,0 +1,30 @@
+# S3etag
+
+calculate AWS S3 etag by multi upload
+
+# Install
+
+    % gem install s3etag
+
+# Gem
+
+    require 's3etag'
+
+    p S3Etag.calc(:data => 'a' * 1000)
+    p S3Etag.calc(:data => 'a' * 1000, :threshold => 100, :min_part_size => 100)
+    p S3Etag.calc(:file => 'test.txt')
+    p S3Etag.calc(:file => '.text.text', :threshold => 100, :min_part_size => 100)
+
+# Command line
+
+    % s3etag
+    s3etag file
+        -t, --threshold threshold
+        -p, --max-parts max-parts
+        -s, --min_part_size min_part_size
+
+    % s3etag text.txt
+    91fdac689d4861c9ae7a0afa21a1f6b8-18
+
+    % s3etag -s 10000000 text.text
+    32aafcd9748824e559b4dbd6b908f6fa-10

data/Rakefile

@@ -0,0 +1,15 @@
+# require 'bundler/gem_tasks'
+
+require 'rake/testtask'
+
+Rake::TestTask.new("test") do |t|
+  t.libs   << "test"
+  t.pattern = "test/**/*_test.rb"
+  # t.verbose = true
+end
+
+desc 'update gem'
+task :update do
+  sh 'gem build s3etag.gemspec'
+  sh 'mv *.gem pkg'
+end

data/bin/s3etag

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 's3etag'
+
+opt = {}
+op = OptionParser.new
+op.on('-t', '--threshold threshold'){ |v| opt[:threshold] = v.to_i }
+op.on('-p', '--max-parts max-parts'){ |v| opt[:max_parts] = v.to_i }
+op.on('-s', '--min_part_size min_part_size'){ |v| opt[:min_part_size] = v.to_i }
+
+path = ARGV.last
+if path && File.file?(path)
+  begin
+    op.parse ARGV
+  rescue OptionParser::InvalidOption => e
+    puts op.summarize(['invalid option.', 's3etag file'], 100)
+  else
+    puts S3Etag.calc(opt.merge(:file => path))
+  end
+else
+  puts op.summarize(['s3etag file'], 100)
+end
+

data/lib/s3etag.rb

@@ -0,0 +1,44 @@
+#!/usr/bin/env ruby
+
+require 'digest/md5'
+require 'optparse'
+require 'stringio'
+
+module S3Etag
+
+  def calc opt = {}
+    threshold = opt[:threshold] || 16 * 1024 * 1024
+    max_parts = opt[:max_parts] || 10000
+    min_part_size = opt[:min_part_size] || 5 * 1024 * 1024
+
+    if opt[:file]
+      total_size = File.stat(opt[:file]).size
+      io = open(opt[:file])
+    elsif opt[:data]
+      total_size = opt[:data].size
+      io = StringIO.new opt[:data]
+    else
+      raise ':file or :data is required'
+    end
+
+    r = nil
+    begin
+      if total_size <= threshold
+        r = Digest::MD5.hexdigest(io.read)
+      else
+        part_size = [(total_size.to_f / max_parts).ceil, min_part_size].max.to_i
+        md5s = []
+        while !io.eof?
+          md5s << Digest::MD5.digest(io.read(part_size))
+        end
+        r = Digest::MD5.hexdigest(md5s.join('')) + '-' + md5s.size.to_s
+      end
+    rescue Exception => e
+      raise e
+    ensure
+      io.close
+    end
+  end
+
+  module_function :calc
+end

data/s3etag.gemspec

@@ -0,0 +1,16 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "s3etag"
+  s.version     = '0.0.1'
+  s.authors     = ["swdyh"]
+  s.homepage    = "https://github.com/swdyh/s3etag"
+  s.summary     = %q{calculate AWS S3 etag by multi upload}
+  s.description = %q{calculate AWS S3 etag by multi upload}
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables = ['s3etag']
+  s.require_paths = ["lib"]
+end

data/test/s3etag_test.rb

@@ -0,0 +1,30 @@
+require 'test/unit'
+require 'digest/md5'
+require 'fileutils'
+require 'pathname'
+require Pathname.new(__FILE__).realpath.join('..', '..', 'lib', 's3etag').to_s
+
+class S3EtagTest < Test::Unit::TestCase
+
+  def test_calc
+    d1 = '0'
+    d2 = '0' * 1024 * 1024 * 16
+    d3 = '0' * (1024 * 1024 * 16 + 1)
+
+    tmp_dir = '_tmp'
+    FileUtils.mkdir_p tmp_dir
+    open(File.join(tmp_dir, 'd1.txt'), 'w') { |f| f.write(d1) }
+    open(File.join(tmp_dir,'d2.txt'), 'w') { |f| f.write(d2) }
+    open(File.join(tmp_dir,'d3.txt'), 'w') { |f| f.write(d3) }
+
+
+    assert_equal Digest::MD5.hexdigest(d1), S3Etag.calc(:data => d1)
+    assert_equal '50838bbf29aec4c6e62bee320d5c9138', S3Etag.calc(:data => d2)
+    assert_equal 'f41a8bb2924e4dccba846428d5ccd921-4', S3Etag.calc(:data => d3)
+    assert_equal Digest::MD5.hexdigest(d1), S3Etag.calc(:file => File.join(tmp_dir, 'd1.txt'))
+    assert_equal '50838bbf29aec4c6e62bee320d5c9138', S3Etag.calc(:file => File.join(tmp_dir, 'd2.txt'))
+    assert_equal 'f41a8bb2924e4dccba846428d5ccd921-4', S3Etag.calc(:file => File.join(tmp_dir,  'd3.txt'))
+
+    FileUtils.rm_rf tmp_dir
+  end
+end

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification 
+name: s3etag
+version: !ruby/object:Gem::Version 
+  hash: 29
+  prerelease: false
+  segments: 
+  - 0
+  - 0
+  - 1
+  version: 0.0.1
+platform: ruby
+authors: 
+- swdyh
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2012-06-26 00:00:00 +09:00
+default_executable: 
+dependencies: []
+
+description: calculate AWS S3 etag by multi upload
+email: 
+executables: 
+- s3etag
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- README.md
+- Rakefile
+- bin/s3etag
+- lib/s3etag.rb
+- s3etag.gemspec
+- test/s3etag_test.rb
+has_rdoc: true
+homepage: https://github.com/swdyh/s3etag
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      hash: 3
+      segments: 
+      - 0
+      version: "0"
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.3.7
+signing_key: 
+specification_version: 3
+summary: calculate AWS S3 etag by multi upload
+test_files: 
+- test/s3etag_test.rb