aliyun-sdk 0.4.1 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 4d2294b9c0a613bb98a3b888d820be1e0c2ecff3
-  data.tar.gz: 2891c2c8687b498e2c404b4637709088975440e9
+  metadata.gz: 36daab0f73cf740bf403be9a9dfdec680bf91e13
+  data.tar.gz: efebb230b85672b7785ddc167b186859ff330d83
 SHA512:
-  metadata.gz: e1ea89fa4af2f7f46a78ada3fda39e7ff491ad7feecc4f8e056791e93f411a403db25076c3bf42a2e2f227e2de7f59e8daf78ee426249d70e6b4182664602ddc
-  data.tar.gz: 61abb3e86fc538213d70826bf2bde4c3e6dbfa47c8cdbd0c5c9eb271359f366930f23c68cc314dcb10cb18c5619fcc140dcebab87c7aa247bf228834a79056b6
+  metadata.gz: 690da0dbe97a706c46a67fd5384ab2ec1c7fec65cfab7c5a7b27bc3e9696f4283ee6314eea6ce00f725b60b6a225f091b936cd8e804482ef005c996ec9e85361
+  data.tar.gz: 19bd7510ef836532f657604e1bd12a1e8e404863a8d317c551bdf7c640740344325a7ca12bb52f5f104d370b8c6f3e93efe7cfe163facbc18168b38f57a403eb

data/CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Change Log
 
+### v0.5.0 / 2016-11-08
+
+- feat: add crc check for uploading(enabled by default) and downloading(disabled by default)
+- bug: fix file open mode for multipart
+
 ### v0.4.1 / 2016-07-19
 
 - Support signature object url with STS

data/ext/crcx/crc64_ecma.c

@@ -0,0 +1,270 @@
+/* crc64.c -- compute CRC-64
+ * Copyright (C) 2013 Mark Adler
+ * Version 1.4  16 Dec 2013  Mark Adler
+ */
+
+/*
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Mark Adler
+  madler@alumni.caltech.edu
+ */
+
+/* Compute CRC-64 in the manner of xz, using the ECMA-182 polynomial,
+   bit-reversed, with one's complement pre and post processing.  Provide a
+   means to combine separately computed CRC-64's. */
+
+/* Version history:
+   1.0  13 Dec 2013  First version
+   1.1  13 Dec 2013  Fix comments in test code
+   1.2  14 Dec 2013  Determine endianess at run time
+   1.3  15 Dec 2013  Add eight-byte processing for big endian as well
+                     Make use of the pthread library optional
+   1.4  16 Dec 2013  Make once variable volatile for limited thread protection
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+#include <assert.h>
+
+/* 64-bit CRC polynomial with these coefficients, but reversed:
+    64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, 35, 33, 32,
+    31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, 4, 1, 0 */
+#define POLY UINT64_C(0xc96c5795d7870f42)
+
+/* Tables for CRC calculation -- filled in by initialization functions that are
+   called once.  These could be replaced by constant tables generated in the
+   same way.  There are two tables, one for each endianess.  Since these are
+   static, i.e. local, one should be compiled out of existence if the compiler
+   can evaluate the endianess check in crc64() at compile time. */
+static uint64_t crc64_little_table[8][256];
+static uint64_t crc64_big_table[8][256];
+
+/* Fill in the CRC-64 constants table. */
+static void crc64_init(uint64_t table[][256])
+{
+    unsigned n, k;
+    uint64_t crc;
+
+    /* generate CRC-64's for all single byte sequences */
+    for (n = 0; n < 256; n++) {
+        crc = n;
+        for (k = 0; k < 8; k++)
+            crc = crc & 1 ? POLY ^ (crc >> 1) : crc >> 1;
+        table[0][n] = crc;
+    }
+
+    /* generate CRC-64's for those followed by 1 to 7 zeros */
+    for (n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (k = 1; k < 8; k++) {
+            crc = table[0][crc & 0xff] ^ (crc >> 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+/* This function is called once to initialize the CRC-64 table for use on a
+   little-endian architecture. */
+static void crc64_little_init(void)
+{
+    crc64_init(crc64_little_table);
+}
+
+/* Reverse the bytes in a 64-bit word. */
+static inline uint64_t rev8(uint64_t a)
+{
+    uint64_t m;
+
+    m = UINT64_C(0xff00ff00ff00ff);
+    a = ((a >> 8) & m) | (a & m) << 8;
+    m = UINT64_C(0xffff0000ffff);
+    a = ((a >> 16) & m) | (a & m) << 16;
+    return a >> 32 | a << 32;
+}
+
+/* This function is called once to initialize the CRC-64 table for use on a
+   big-endian architecture. */
+static void crc64_big_init(void)
+{
+    unsigned k, n;
+
+    crc64_init(crc64_big_table);
+    for (k = 0; k < 8; k++)
+        for (n = 0; n < 256; n++)
+            crc64_big_table[k][n] = rev8(crc64_big_table[k][n]);
+}
+
+/* init table once */
+void crc64_init_once(void)
+{
+    crc64_little_init();
+    crc64_big_init();
+}
+
+/* Calculate a CRC-64 eight bytes at a time on a little-endian architecture. */
+static inline uint64_t crc64_little(uint64_t crc, void *buf, size_t len)
+{
+    unsigned char *next = (unsigned char *) buf;
+
+    crc = ~crc;
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = crc64_little_table[7][crc & 0xff] ^
+              crc64_little_table[6][(crc >> 8) & 0xff] ^
+              crc64_little_table[5][(crc >> 16) & 0xff] ^
+              crc64_little_table[4][(crc >> 24) & 0xff] ^
+              crc64_little_table[3][(crc >> 32) & 0xff] ^
+              crc64_little_table[2][(crc >> 40) & 0xff] ^
+              crc64_little_table[1][(crc >> 48) & 0xff] ^
+              crc64_little_table[0][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+    while (len) {
+        crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    return ~crc;
+}
+
+/* Calculate a CRC-64 eight bytes at a time on a big-endian architecture. */
+static inline uint64_t crc64_big(uint64_t crc, void *buf, size_t len)
+{
+    unsigned char *next = (unsigned char *) buf;
+
+    crc = ~rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = crc64_big_table[0][crc & 0xff] ^
+              crc64_big_table[1][(crc >> 8) & 0xff] ^
+              crc64_big_table[2][(crc >> 16) & 0xff] ^
+              crc64_big_table[3][(crc >> 24) & 0xff] ^
+              crc64_big_table[4][(crc >> 32) & 0xff] ^
+              crc64_big_table[5][(crc >> 40) & 0xff] ^
+              crc64_big_table[6][(crc >> 48) & 0xff] ^
+              crc64_big_table[7][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+    while (len) {
+        crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+    return ~rev8(crc);
+}
+
+/* Return the CRC-64 of buf[0..len-1] with initial crc, processing eight bytes
+   at a time.  This selects one of two routines depending on the endianess of
+   the architecture.  A good optimizing compiler will determine the endianess
+   at compile time if it can, and get rid of the unused code and table.  If the
+   endianess can be changed at run time, then this code will handle that as
+   well, initializing and using two tables, if called upon to do so. */
+uint64_t crc64(uint64_t crc, void *buf, size_t len)
+{
+    uint64_t n = 1;
+
+    return *(char *)&n ? crc64_little(crc, buf, len) :
+                         crc64_big(crc, buf, len);
+}
+
+#define GF2_DIM 64      /* dimension of GF(2) vectors (length of CRC) */
+
+static uint64_t gf2_matrix_times(uint64_t *mat, uint64_t vec)
+{
+    uint64_t sum;
+
+    sum = 0;
+    while (vec) {
+        if (vec & 1)
+            sum ^= *mat;
+        vec >>= 1;
+        mat++;
+    }
+    return sum;
+}
+
+static void gf2_matrix_square(uint64_t *square, uint64_t *mat)
+{
+    unsigned n;
+
+    for (n = 0; n < GF2_DIM; n++)
+        square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+/* Return the CRC-64 of two sequential blocks, where crc1 is the CRC-64 of the
+   first block, crc2 is the CRC-64 of the second block, and len2 is the length
+   of the second block. */
+uint64_t crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2)
+{
+    unsigned n;
+    uint64_t row;
+    uint64_t even[GF2_DIM];     /* even-power-of-two zeros operator */
+    uint64_t odd[GF2_DIM];      /* odd-power-of-two zeros operator */
+
+    /* degenerate case */
+    if (len2 == 0)
+        return crc1;
+
+    /* put operator for one zero bit in odd */
+    odd[0] = POLY;              /* CRC-64 polynomial */
+    row = 1;
+    for (n = 1; n < GF2_DIM; n++) {
+        odd[n] = row;
+        row <<= 1;
+    }
+
+    /* put operator for two zero bits in even */
+    gf2_matrix_square(even, odd);
+
+    /* put operator for four zero bits in odd */
+    gf2_matrix_square(odd, even);
+
+    /* apply len2 zeros to crc1 (first square will put the operator for one
+       zero byte, eight zero bits, in even) */
+    do {
+        /* apply zeros operator for this bit of len2 */
+        gf2_matrix_square(even, odd);
+        if (len2 & 1)
+            crc1 = gf2_matrix_times(even, crc1);
+        len2 >>= 1;
+
+        /* if no more bits set, then done */
+        if (len2 == 0)
+            break;
+
+        /* another iteration of the loop with odd and even swapped */
+        gf2_matrix_square(odd, even);
+        if (len2 & 1)
+            crc1 = gf2_matrix_times(odd, crc1);
+        len2 >>= 1;
+
+        /* if no more bits set, then done */
+    } while (len2 != 0);
+
+    /* return combined crc */
+    crc1 ^= crc2;
+    return crc1;
+}

data/ext/crcx/crcx.c

@@ -0,0 +1,45 @@
+#include "crcx.h"
+
+void Init_crcx(){
+    VALUE mAliyun = Qnil;
+    VALUE CrcX = Qnil;
+
+    crc64_init_once();
+
+    mAliyun = rb_define_module("Aliyun");
+    CrcX = rb_define_module_under(mAliyun, "CrcX");
+    rb_define_module_function(CrcX, "crc64", crc64_wrapper, 3);
+    rb_define_module_function(CrcX, "crc64_combine", crc64_combine_wrapper, 3);
+}
+
+void check_num_type(VALUE crc_value)
+{
+    if (T_BIGNUM == TYPE(crc_value)) {
+        Check_Type(crc_value, T_BIGNUM);
+    }
+    else {
+        Check_Type(crc_value, T_FIXNUM);
+    }
+    return ;
+}
+
+VALUE crc64_wrapper(VALUE self, VALUE init_crc, VALUE buffer, VALUE size)
+{
+    uint64_t crc_value = 0;
+
+    check_num_type(init_crc);
+    check_num_type(size);
+    Check_Type(buffer, T_STRING);
+    crc_value = crc64(NUM2ULL(init_crc), (void *)RSTRING_PTR(buffer), NUM2ULL(size));
+    return ULL2NUM(crc_value);
+}
+
+VALUE crc64_combine_wrapper(VALUE self, VALUE crc1, VALUE crc2, VALUE len2)
+{
+    uint64_t crc_value = 0;
+    check_num_type(crc1);
+    check_num_type(crc2);
+    check_num_type(len2);
+    crc_value = crc64_combine(NUM2ULL(crc1), NUM2ULL(crc2), NUM2ULL(len2));
+    return ULL2NUM(crc_value);
+}

data/ext/crcx/crcx.h

@@ -0,0 +1,8 @@
+#include <ruby.h>
+
+uint64_t crc64(uint64_t crc, void *buf, size_t len);
+uint64_t crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2);
+void crc64_init_once(void);
+
+VALUE crc64_wrapper(VALUE self, VALUE init_crc, VALUE buffer, VALUE size);
+VALUE crc64_combine_wrapper(VALUE self, VALUE crc1, VALUE crc2, VALUE len2);

data/ext/crcx/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile('aliyun/crcx')

data/lib/aliyun/oss.rb

@@ -14,3 +14,4 @@ require_relative 'oss/iterator'
 require_relative 'oss/object'
 require_relative 'oss/bucket'
 require_relative 'oss/client'
+require_relative 'crcx'

data/lib/aliyun/oss/bucket.rb

@@ -639,6 +639,18 @@ module Aliyun
         @protocol.sign(string_to_sign)
       end
 
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @protocol.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @protocol.upload_crc_enable
+      end
+
       private
       # Infer the file's content type using MIME::Types
       # @param file [String] the file path
@@ -655,7 +667,6 @@ module Aliyun
       def get_cpt_file(file)
         "#{File.expand_path(file)}.cpt"
       end
-
     end # Bucket
   end # OSS
 end # Aliyun

data/lib/aliyun/oss/client.rb

@@ -28,6 +28,10 @@ module Aliyun
       #  KEY SECRET，如果不填则会尝试匿名访问
       # @option opts [Boolean] :cname [可选] 指定endpoint是否是用户绑
       #  定的域名
+      # @option opts [Boolean] :upload_crc_enable [可选]指定上传处理
+      #  是否开启CRC校验，默认为开启(true)
+      # @option opts [Boolean] :download_crc_enable [可选]指定下载处理
+      #  是否开启CRC校验，默认为不开启(false)
       # @option opts [String] :sts_token [可选] 指定STS的
       #  SecurityToken，如果指定，则使用STS授权访问
       # @option opts [Fixnum] :open_timeout [可选] 指定建立连接的超时

data/lib/aliyun/oss/config.rb

@@ -11,7 +11,8 @@ module Aliyun
 
       attrs :endpoint, :cname, :sts_token,
             :access_key_id, :access_key_secret,
-            :open_timeout, :read_timeout
+            :open_timeout, :read_timeout,
+            :download_crc_enable, :upload_crc_enable
 
       def initialize(opts = {})
         super(opts)
@@ -19,6 +20,8 @@ module Aliyun
         @access_key_id = @access_key_id.strip if @access_key_id
         @access_key_secret = @access_key_secret.strip if @access_key_secret
         normalize_endpoint if endpoint
+        @upload_crc_enable = (@upload_crc_enable == 'false' || @upload_crc_enable == false) ? false : true
+        @download_crc_enable = (@download_crc_enable == 'true' || @download_crc_enable == true) ? true : false
       end
 
       private

data/lib/aliyun/oss/download.rb

@@ -114,7 +114,7 @@ module Aliyun
 
           parts = sync_get_all_parts
           # concat all part files into the target file
-          File.open(@file, 'w') do |w|
+          File.open(@file, 'wb') do |w|
             parts.sort{ |x, y| x[:number] <=> y[:number] }.each do |p|
               File.open(get_part_file(p)) do |r|
                   w.write(r.read(READ_SIZE)) until r.eof?
@@ -177,7 +177,7 @@ module Aliyun
           logger.debug("Begin download part: #{p}")
 
           part_file = get_part_file(p)
-          File.open(part_file, 'w') do |w|
+          File.open(part_file, 'wb') do |w|
             @protocol.get_object(
               bucket, object,
               @options.merge(range: p[:range])) { |chunk| w.write(chunk) }

data/lib/aliyun/oss/exception.rb

@@ -64,6 +64,12 @@ module Aliyun
     class ClientError < Common::Exception
     end # ClientError
 
+    ##
+    # CrcInconsistentError will be raised after a upload operation,
+    # when the local crc is inconsistent with the response crc from server. 
+    #
+    class CrcInconsistentError < Common::Exception; end
+
     ##
     # FileInconsistentError happens in a resumable upload transaction,
     # when the file to upload has changed during the uploading