aliyun-sdk 0.4.1 → 0.5.0

data/spec/aliyun/oss/util_spec.rb

@@ -44,6 +44,56 @@ module Aliyun
         expect(signature).to eq("7Oh2wobzeg6dw/cWYbF/2m6s6qc=")
       end
 
+      # 测试CRC计算是否正确
+      it "should calculate a correct data crc" do
+        content = ""
+        crc = Util.crc(content)
+        expect(crc).to eq(0)
+
+        content = "hello world"
+        crc = Util.crc(content)
+        expect(crc).to eq(5981764153023615706)
+
+        content = "test\0hello\1world\2!\3"
+        crc = Util.crc(content)
+        expect(crc).to eq(6745424696046691431)
+      end
+
+      # 测试CRC Combine计算是否正确
+      it "should calculate a correct crc that crc_a combine with crc_b" do
+        content_a = "test\0hello\1world\2!\3"
+        crc_a = Util.crc(content_a)
+        expect(crc_a).to eq(6745424696046691431)
+
+        content_b = "hello world"
+        crc_b = Util.crc(content_b)
+        expect(crc_b).to eq(5981764153023615706)
+
+        crc_c = Util.crc_combine(crc_a, crc_b, content_b.size)
+        expect(crc_c).to eq(13027479509578346683)
+
+        crc_ab = Util.crc(content_a + content_b)
+        expect(crc_ab).to eq(crc_c)
+
+        crc_ab = Util.crc(content_b, crc_a)
+        expect(crc_ab).to eq(crc_c)
+      end
+
+      # 测试CRC校验和异常处理是否正确
+      it "should check inconsistent crc" do
+        expect {
+          Util.crc_check(6745424696046691431, 6745424696046691431, 'put')
+        }.not_to raise_error
+        
+        expect {
+          Util.crc_check(6745424696046691431, 5981764153023615706, 'append')
+        }.to raise_error(CrcInconsistentError, "The crc of append between client and oss is not inconsistent.")
+
+        expect {
+          Util.crc_check(6745424696046691431, -1, 'post')
+        }.to raise_error(CrcInconsistentError, "The crc of post between client and oss is not inconsistent.")
+      end
+
     end # Util
 
   end # OSS

data/tests/config.rb

@@ -4,6 +4,8 @@ class TestConf
       {
         access_key_id: ENV['RUBY_SDK_OSS_ID'],
         access_key_secret: ENV['RUBY_SDK_OSS_KEY'],
+        download_crc_enable: ENV['RUBY_SDK_OSS_DOWNLOAD_CRC_ENABLE'],
+        upload_crc_enable: ENV['RUBY_SDK_OSS_UPLOAD_CRC_ENABLE'],
         endpoint: ENV['RUBY_SDK_OSS_ENDPOINT']
       }
     end

data/tests/helper.rb

@@ -0,0 +1,15 @@
+# -*- encoding: utf-8 -*-
+
+module Aliyun
+  module Test
+    module Helper
+      def random_string(n)
+        (1...n).map { (65 + rand(26)).chr }.join + "\n"
+      end
+
+      def random_bytes(n)
+        (1...n).map { rand(255).chr }.join + "\n"
+      end
+    end
+  end
+end

data/tests/test_crc_check.rb

@@ -0,0 +1,184 @@
+require 'minitest/autorun'
+require 'yaml'
+require 'tempfile'
+$LOAD_PATH.unshift(File.expand_path("../../lib", __FILE__))
+require 'aliyun/oss'
+require_relative 'config'
+require_relative 'helper'
+
+class TestCrcCheck < Minitest::Test
+
+  include Aliyun::Test::Helper
+  
+  @@tests_run = 0
+  @@test_file = nil
+
+  def setup
+    Aliyun::Common::Logging.set_log_level(Logger::DEBUG)
+    client = Aliyun::OSS::Client.new(TestConf.creds)
+    @bucket = client.get_bucket(TestConf.bucket)
+    @prefix = 'tests/crc_check/'
+
+    if @@tests_run == 0
+      @@test_file = Tempfile.new('oss_ruby_sdk_test_crc')
+      (10 * 1024).times { @@test_file.write(random_bytes(1024)) }
+    end
+    @@tests_run += 1
+  end
+
+  def teardown
+    if @@tests_run == TestCrcCheck.runnable_methods.length
+      @@test_file.unlink unless @@test_file.nil?
+    end
+  end
+
+  def get_key(k)
+    @prefix + k
+  end
+
+  def test_put_object
+    skip unless TestConf.creds[:upload_crc_enable]
+
+    # Check crc status
+    assert(@bucket.upload_crc_enable)
+
+    # Create a test file with 10MB random bytes to put.
+    key = get_key('put_file')
+    
+    @bucket.put_object(key, :file => @@test_file.path)
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 10 * 1024 * 1024)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.put_object(key, {:init_crc => 1, :file => @@test_file.path}) do |content|
+        content << 'hello world.'
+      end
+    end
+
+    # Put a string to oss.
+    key = get_key('put_string')
+    @bucket.put_object(key, :init_crc => 0) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.put_object(key, :init_crc => 1) do |content|
+        content << 'hello world.'
+      end
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_append_object
+    skip unless TestConf.creds[:upload_crc_enable]
+    key = get_key('append_file')
+
+    # Check crc status
+    assert(@bucket.upload_crc_enable)
+
+    # Check $key object doesn't exist.
+    test_object = @bucket.get_object(key) rescue 0
+    @bucket.delete_object(key) if test_object.size
+
+    # Create a append object to oss with a string.
+    position = @bucket.append_object(key, 0, :init_crc => 0) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Append a test file to oss $key object.
+    @bucket.append_object(key, position, {:init_crc => test_object.headers[:x_oss_hash_crc64ecma], :file => @@test_file.path})
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size + (10 * 1024 * 1024))
+
+    # No crc check when init_crc is nil
+    position = @bucket.append_object(key, test_object.size) do |content|
+      content << 'hello world.'
+    end
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, 'hello world.'.size * 2 + (10 * 1024 * 1024))
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      position = @bucket.append_object(key, test_object.size, :init_crc => 0) do |content|
+        content << 'hello world.'
+      end
+    end
+
+    # Check crc wrong case.
+    test_object = @bucket.get_object(key)
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.append_object(key, test_object.size, {:init_crc => 0, :file => @@test_file.path})
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_upload_object
+    skip unless TestConf.creds[:upload_crc_enable]
+    key = get_key('upload_file')
+
+    # Check crc status
+    assert(@bucket.upload_crc_enable)
+    @bucket.resumable_upload(key, @@test_file.path, :cpt_file => "#{@@test_file.path}.cpt", threads: 2, :part_size => 1024 * 1024)
+
+    test_object = @bucket.get_object(key)
+    assert_equal(test_object.size, (10 * 1024 * 1024))
+
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_get_small_object
+    skip unless TestConf.creds[:download_crc_enable]
+
+    # Check crc status
+    assert(@bucket.download_crc_enable)
+
+    # Put a string to oss.
+    key = get_key('get_small_object')
+    @bucket.put_object(key) do |content|
+      content << 'hello world.'
+    end
+    temp_buf = ""
+    test_object = @bucket.get_object(key) { |c| temp_buf << c }
+    assert_equal(test_object.size, 'hello world.'.size)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.get_object(key, {:init_crc => 1}) { |c| temp_buf << c }
+    end
+  ensure
+    @bucket.delete_object(key)
+  end
+
+  def test_get_large_object
+    skip unless TestConf.creds[:download_crc_enable]
+
+    # Check crc status
+    assert(@bucket.download_crc_enable)
+
+    # put a test file with 10MB random bytes to oss for testing get.
+    key = get_key('get_file')
+    @bucket.put_object(key, :file => @@test_file.path)
+
+    get_temp_file = Tempfile.new('oss_ruby_sdk_test_crc_get')
+    test_object = @bucket.get_object(key, {:file => get_temp_file})
+    assert_equal(test_object.size, 10 * 1024 * 1024)
+
+    # Check crc wrong case.
+    assert_raises Aliyun::OSS::CrcInconsistentError do
+      @bucket.get_object(key, {:file => get_temp_file, :init_crc => 1})
+    end
+  ensure
+    get_temp_file.unlink unless get_temp_file.nil?
+    @bucket.delete_object(key)
+  end
+
+end

data/tests/test_custom_headers.rb

@@ -36,6 +36,7 @@ class TestCustomHeaders < Minitest::Test
 
     content_encoding = 'deflate'
     expires = (Time.now + 3600).httpdate
+
     @bucket.put_object(
       key,
       headers: {'cache-control' => cache_control,
@@ -46,12 +47,19 @@ class TestCustomHeaders < Minitest::Test
     end
 
     content = ''
-    obj = @bucket.get_object(key) { |c| content << c }
-    assert_equal 'hello world', content
-    assert_equal cache_control, obj.headers[:cache_control]
-    assert_equal content_disposition, obj.headers[:content_disposition]
-    assert_equal content_encoding, obj.headers[:content_encoding]
-    assert_equal expires, obj.headers[:expires]
+    obj = nil
+    if @bucket.download_crc_enable
+      assert_raises Aliyun::OSS::CrcInconsistentError do
+        obj = @bucket.get_object(key) { |c| content << c }
+      end
+    else
+      obj = @bucket.get_object(key) { |c| content << c }
+      assert_equal 'hello world', content
+      assert_equal cache_control, obj.headers[:cache_control]
+      assert_equal content_disposition, obj.headers[:content_disposition]
+      assert_equal content_encoding, obj.headers[:content_encoding]
+      assert_equal expires, obj.headers[:expires]
+    end
   end
 
   def test_headers_overwrite

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: aliyun-sdk
 version: !ruby/object:Gem::Version
-  version: 0.4.1
+  version: 0.5.0
 platform: ruby
 authors:
 - Tianlong Wu
 autorequire: 
-bindir: bin
+bindir: lib/aliyun
 cert_chain: []
-date: 2016-07-19 00:00:00.000000000 Z
+date: 2016-11-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -66,6 +66,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '10.4'
+- !ruby/object:Gem::Dependency
+  name: rake-compiler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.9.0
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -126,7 +140,8 @@ description: A Ruby program to facilitate accessing Aliyun Object Storage Servic
 email:
 - rockuw.@gmail.com
 executables: []
-extensions: []
+extensions:
+- ext/crcx/extconf.rb
 extra_rdoc_files:
 - README.md
 - CHANGELOG.md
@@ -141,6 +156,10 @@ files:
 - examples/aliyun/oss/streaming.rb
 - examples/aliyun/oss/using_sts.rb
 - examples/aliyun/sts/assume_role.rb
+- ext/crcx/crc64_ecma.c
+- ext/crcx/crcx.c
+- ext/crcx/crcx.h
+- ext/crcx/extconf.rb
 - lib/aliyun/common.rb
 - lib/aliyun/common/exception.rb
 - lib/aliyun/common/logging.rb
@@ -180,8 +199,10 @@ files:
 - spec/aliyun/sts/client_spec.rb
 - spec/aliyun/sts/util_spec.rb
 - tests/config.rb
+- tests/helper.rb
 - tests/test_content_encoding.rb
 - tests/test_content_type.rb
+- tests/test_crc_check.rb
 - tests/test_custom_headers.rb
 - tests/test_encoding.rb
 - tests/test_large_file.rb
@@ -228,8 +249,10 @@ test_files:
 - spec/aliyun/sts/client_spec.rb
 - spec/aliyun/sts/util_spec.rb
 - tests/config.rb
+- tests/helper.rb
 - tests/test_content_encoding.rb
 - tests/test_content_type.rb
+- tests/test_crc_check.rb
 - tests/test_custom_headers.rb
 - tests/test_encoding.rb
 - tests/test_large_file.rb