aliyun-sdk 0.4.1 → 0.7.2

data/ext/crcx/crc64_ecma.c

@@ -0,0 +1,270 @@
+/* crc64.c -- compute CRC-64
+ * Copyright (C) 2013 Mark Adler
+ * Version 1.4  16 Dec 2013  Mark Adler
+ */
+
+/*
+  This software is provided 'as-is', without any express or implied
+  warranty.  In no event will the author be held liable for any damages
+  arising from the use of this software.
+
+  Permission is granted to anyone to use this software for any purpose,
+  including commercial applications, and to alter it and redistribute it
+  freely, subject to the following restrictions:
+
+  1. The origin of this software must not be misrepresented; you must not
+     claim that you wrote the original software. If you use this software
+     in a product, an acknowledgment in the product documentation would be
+     appreciated but is not required.
+  2. Altered source versions must be plainly marked as such, and must not be
+     misrepresented as being the original software.
+  3. This notice may not be removed or altered from any source distribution.
+
+  Mark Adler
+  madler@alumni.caltech.edu
+ */
+
+/* Compute CRC-64 in the manner of xz, using the ECMA-182 polynomial,
+   bit-reversed, with one's complement pre and post processing.  Provide a
+   means to combine separately computed CRC-64's. */
+
+/* Version history:
+   1.0  13 Dec 2013  First version
+   1.1  13 Dec 2013  Fix comments in test code
+   1.2  14 Dec 2013  Determine endianess at run time
+   1.3  15 Dec 2013  Add eight-byte processing for big endian as well
+                     Make use of the pthread library optional
+   1.4  16 Dec 2013  Make once variable volatile for limited thread protection
+ */
+
+#include <stdio.h>
+#include <inttypes.h>
+#include <assert.h>
+
+/* 64-bit CRC polynomial with these coefficients, but reversed:
+    64, 62, 57, 55, 54, 53, 52, 47, 46, 45, 40, 39, 38, 37, 35, 33, 32,
+    31, 29, 27, 24, 23, 22, 21, 19, 17, 13, 12, 10, 9, 7, 4, 1, 0 */
+#define POLY UINT64_C(0xc96c5795d7870f42)
+
+/* Tables for CRC calculation -- filled in by initialization functions that are
+   called once.  These could be replaced by constant tables generated in the
+   same way.  There are two tables, one for each endianess.  Since these are
+   static, i.e. local, one should be compiled out of existence if the compiler
+   can evaluate the endianess check in crc64() at compile time. */
+static uint64_t crc64_little_table[8][256];
+static uint64_t crc64_big_table[8][256];
+
+/* Fill in the CRC-64 constants table. */
+static void crc64_init(uint64_t table[][256])
+{
+    unsigned n, k;
+    uint64_t crc;
+
+    /* generate CRC-64's for all single byte sequences */
+    for (n = 0; n < 256; n++) {
+        crc = n;
+        for (k = 0; k < 8; k++)
+            crc = crc & 1 ? POLY ^ (crc >> 1) : crc >> 1;
+        table[0][n] = crc;
+    }
+
+    /* generate CRC-64's for those followed by 1 to 7 zeros */
+    for (n = 0; n < 256; n++) {
+        crc = table[0][n];
+        for (k = 1; k < 8; k++) {
+            crc = table[0][crc & 0xff] ^ (crc >> 8);
+            table[k][n] = crc;
+        }
+    }
+}
+
+/* This function is called once to initialize the CRC-64 table for use on a
+   little-endian architecture. */
+static void crc64_little_init(void)
+{
+    crc64_init(crc64_little_table);
+}
+
+/* Reverse the bytes in a 64-bit word. */
+static inline uint64_t rev8(uint64_t a)
+{
+    uint64_t m;
+
+    m = UINT64_C(0xff00ff00ff00ff);
+    a = ((a >> 8) & m) | (a & m) << 8;
+    m = UINT64_C(0xffff0000ffff);
+    a = ((a >> 16) & m) | (a & m) << 16;
+    return a >> 32 | a << 32;
+}
+
+/* This function is called once to initialize the CRC-64 table for use on a
+   big-endian architecture. */
+static void crc64_big_init(void)
+{
+    unsigned k, n;
+
+    crc64_init(crc64_big_table);
+    for (k = 0; k < 8; k++)
+        for (n = 0; n < 256; n++)
+            crc64_big_table[k][n] = rev8(crc64_big_table[k][n]);
+}
+
+/* init table once */
+void crc64_init_once(void)
+{
+    crc64_little_init();
+    crc64_big_init();
+}
+
+/* Calculate a CRC-64 eight bytes at a time on a little-endian architecture. */
+static inline uint64_t crc64_little(uint64_t crc, void *buf, size_t len)
+{
+    unsigned char *next = (unsigned char *) buf;
+
+    crc = ~crc;
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = crc64_little_table[7][crc & 0xff] ^
+              crc64_little_table[6][(crc >> 8) & 0xff] ^
+              crc64_little_table[5][(crc >> 16) & 0xff] ^
+              crc64_little_table[4][(crc >> 24) & 0xff] ^
+              crc64_little_table[3][(crc >> 32) & 0xff] ^
+              crc64_little_table[2][(crc >> 40) & 0xff] ^
+              crc64_little_table[1][(crc >> 48) & 0xff] ^
+              crc64_little_table[0][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+    while (len) {
+        crc = crc64_little_table[0][(crc ^ *next++) & 0xff] ^ (crc >> 8);
+        len--;
+    }
+    return ~crc;
+}
+
+/* Calculate a CRC-64 eight bytes at a time on a big-endian architecture. */
+static inline uint64_t crc64_big(uint64_t crc, void *buf, size_t len)
+{
+    unsigned char *next = (unsigned char *) buf;
+
+    crc = ~rev8(crc);
+    while (len && ((uintptr_t)next & 7) != 0) {
+        crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+    while (len >= 8) {
+        crc ^= *(uint64_t *)next;
+        crc = crc64_big_table[0][crc & 0xff] ^
+              crc64_big_table[1][(crc >> 8) & 0xff] ^
+              crc64_big_table[2][(crc >> 16) & 0xff] ^
+              crc64_big_table[3][(crc >> 24) & 0xff] ^
+              crc64_big_table[4][(crc >> 32) & 0xff] ^
+              crc64_big_table[5][(crc >> 40) & 0xff] ^
+              crc64_big_table[6][(crc >> 48) & 0xff] ^
+              crc64_big_table[7][crc >> 56];
+        next += 8;
+        len -= 8;
+    }
+    while (len) {
+        crc = crc64_big_table[0][(crc >> 56) ^ *next++] ^ (crc << 8);
+        len--;
+    }
+    return ~rev8(crc);
+}
+
+/* Return the CRC-64 of buf[0..len-1] with initial crc, processing eight bytes
+   at a time.  This selects one of two routines depending on the endianess of
+   the architecture.  A good optimizing compiler will determine the endianess
+   at compile time if it can, and get rid of the unused code and table.  If the
+   endianess can be changed at run time, then this code will handle that as
+   well, initializing and using two tables, if called upon to do so. */
+uint64_t crc64(uint64_t crc, void *buf, size_t len)
+{
+    uint64_t n = 1;
+
+    return *(char *)&n ? crc64_little(crc, buf, len) :
+                         crc64_big(crc, buf, len);
+}
+
+#define GF2_DIM 64      /* dimension of GF(2) vectors (length of CRC) */
+
+static uint64_t gf2_matrix_times(uint64_t *mat, uint64_t vec)
+{
+    uint64_t sum;
+
+    sum = 0;
+    while (vec) {
+        if (vec & 1)
+            sum ^= *mat;
+        vec >>= 1;
+        mat++;
+    }
+    return sum;
+}
+
+static void gf2_matrix_square(uint64_t *square, uint64_t *mat)
+{
+    unsigned n;
+
+    for (n = 0; n < GF2_DIM; n++)
+        square[n] = gf2_matrix_times(mat, mat[n]);
+}
+
+/* Return the CRC-64 of two sequential blocks, where crc1 is the CRC-64 of the
+   first block, crc2 is the CRC-64 of the second block, and len2 is the length
+   of the second block. */
+uint64_t crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2)
+{
+    unsigned n;
+    uint64_t row;
+    uint64_t even[GF2_DIM];     /* even-power-of-two zeros operator */
+    uint64_t odd[GF2_DIM];      /* odd-power-of-two zeros operator */
+
+    /* degenerate case */
+    if (len2 == 0)
+        return crc1;
+
+    /* put operator for one zero bit in odd */
+    odd[0] = POLY;              /* CRC-64 polynomial */
+    row = 1;
+    for (n = 1; n < GF2_DIM; n++) {
+        odd[n] = row;
+        row <<= 1;
+    }
+
+    /* put operator for two zero bits in even */
+    gf2_matrix_square(even, odd);
+
+    /* put operator for four zero bits in odd */
+    gf2_matrix_square(odd, even);
+
+    /* apply len2 zeros to crc1 (first square will put the operator for one
+       zero byte, eight zero bits, in even) */
+    do {
+        /* apply zeros operator for this bit of len2 */
+        gf2_matrix_square(even, odd);
+        if (len2 & 1)
+            crc1 = gf2_matrix_times(even, crc1);
+        len2 >>= 1;
+
+        /* if no more bits set, then done */
+        if (len2 == 0)
+            break;
+
+        /* another iteration of the loop with odd and even swapped */
+        gf2_matrix_square(odd, even);
+        if (len2 & 1)
+            crc1 = gf2_matrix_times(odd, crc1);
+        len2 >>= 1;
+
+        /* if no more bits set, then done */
+    } while (len2 != 0);
+
+    /* return combined crc */
+    crc1 ^= crc2;
+    return crc1;
+}

data/ext/crcx/crcx.c

@@ -0,0 +1,45 @@
+#include "crcx.h"
+
+void Init_crcx(){
+    VALUE mAliyun = Qnil;
+    VALUE CrcX = Qnil;
+
+    crc64_init_once();
+
+    mAliyun = rb_define_module("Aliyun");
+    CrcX = rb_define_module_under(mAliyun, "CrcX");
+    rb_define_module_function(CrcX, "crc64", crc64_wrapper, 3);
+    rb_define_module_function(CrcX, "crc64_combine", crc64_combine_wrapper, 3);
+}
+
+void check_num_type(VALUE crc_value)
+{
+    if (T_BIGNUM == TYPE(crc_value)) {
+        Check_Type(crc_value, T_BIGNUM);
+    }
+    else {
+        Check_Type(crc_value, T_FIXNUM);
+    }
+    return ;
+}
+
+VALUE crc64_wrapper(VALUE self, VALUE init_crc, VALUE buffer, VALUE size)
+{
+    uint64_t crc_value = 0;
+
+    check_num_type(init_crc);
+    check_num_type(size);
+    Check_Type(buffer, T_STRING);
+    crc_value = crc64(NUM2ULL(init_crc), (void *)RSTRING_PTR(buffer), NUM2ULL(size));
+    return ULL2NUM(crc_value);
+}
+
+VALUE crc64_combine_wrapper(VALUE self, VALUE crc1, VALUE crc2, VALUE len2)
+{
+    uint64_t crc_value = 0;
+    check_num_type(crc1);
+    check_num_type(crc2);
+    check_num_type(len2);
+    crc_value = crc64_combine(NUM2ULL(crc1), NUM2ULL(crc2), NUM2ULL(len2));
+    return ULL2NUM(crc_value);
+}

data/ext/crcx/crcx.h

@@ -0,0 +1,8 @@
+#include <ruby.h>
+
+uint64_t crc64(uint64_t crc, void *buf, size_t len);
+uint64_t crc64_combine(uint64_t crc1, uint64_t crc2, uintmax_t len2);
+void crc64_init_once(void);
+
+VALUE crc64_wrapper(VALUE self, VALUE init_crc, VALUE buffer, VALUE size);
+VALUE crc64_combine_wrapper(VALUE self, VALUE crc1, VALUE crc2, VALUE len2);

data/ext/crcx/extconf.rb

@@ -0,0 +1,3 @@
+require 'mkmf'
+
+create_makefile('aliyun/crcx')

data/lib/aliyun/common/logging.rb

@@ -34,8 +34,13 @@ module Aliyun
 
       def self.logger
         unless @logger
+          @log_file = nil
+          # Environment parameter ALIYUN_OSS_SDK_LOG_PATH used to control whether output log to a file
+          if ENV['ALIYUN_OSS_SDK_LOG_PATH']
+            @log_file ||= DEFAULT_LOG_FILE
+          end
           @logger = Logger.new(
-            @log_file ||= DEFAULT_LOG_FILE, MAX_NUM_LOG, ROTATE_SIZE)
+              @log_file, MAX_NUM_LOG, ROTATE_SIZE)
           @logger.level = Logger::INFO
         end
         @logger

data/lib/aliyun/oss.rb

@@ -14,3 +14,4 @@ require_relative 'oss/iterator'
 require_relative 'oss/object'
 require_relative 'oss/bucket'
 require_relative 'oss/client'
+require_relative 'crcx'

data/lib/aliyun/oss/bucket.rb

@@ -588,42 +588,39 @@ module Aliyun
       # 获取Object的URL
       # @param [String] key Object的key
       # @param [Boolean] sign 是否对URL进行签名，默认为是
-      # @param [Fixnum] expiry URL的有效时间，单位为秒，默认为60s
+      # @param [Integer] expiry URL的有效时间，单位为秒，默认为60s
+      # @param [Hash] parameters 附加的query参数，默认为空
       # @return [String] 用于直接访问Object的URL
-      def object_url(key, sign = true, expiry = 60)
-        url = @protocol.get_request_url(name, key)
-        return url unless sign
-
-        expires = Time.now.to_i + expiry
-        query = {
-          'Expires' => expires.to_s,
-          'OSSAccessKeyId' => CGI.escape(access_key_id)
-        }
-
-        sub_res = []
-        if @protocol.get_sts_token
-          sub_res << "security-token=#{@protocol.get_sts_token}"
-          query['security-token'] = CGI.escape(@protocol.get_sts_token)
-        end
-
-        resource = "/#{name}/#{key}"
-        unless sub_res.empty?
-          resource << "?#{sub_res.join('&')}"
-        end
+      def object_url(key, sign = true, expiry = 60, parameters = {})
+        url = @protocol.get_request_url(name, key).gsub('%2F', '/')
+        query = parameters.dup
+
+        if sign
+          #header
+          expires = Time.now.to_i + expiry
+          headers = {
+            'date' => expires.to_s,
+          }
+
+          #query 
+          if @protocol.get_sts_token
+            query['security-token'] = @protocol.get_sts_token
+          end
 
-        string_to_sign = "" <<
-                         "GET\n" << # method
-                         "\n" <<    # Content-MD5
-                         "\n" <<    # Content-Type
-                         "#{expires}\n" <<
-                         "#{resource}"
+          res = {
+            :path => @protocol.get_resource_path(name, key),
+            :sub_res => query,
+          }
+          signature = Util.get_signature(@protocol.get_access_key_secret, 'GET', headers, res)
 
-        signature = sign(string_to_sign)
-        query_string =
-          query.merge('Signature' => CGI.escape(signature))
-          .map { |k, v| "#{k}=#{v}" }.join('&')
+          query['Expires'] = expires.to_s
+          query['OSSAccessKeyId'] = @protocol.get_access_key_id
+          query['Signature'] = signature
+        end  
 
-        [url, query_string].join('?')
+        query_string = query.map { |k, v| v ? [k, CGI.escape(v)].join("=") : k }.join("&")
+        link_char = query_string.empty? ? '' : '?'
+        [url, query_string].join(link_char)
       end
 
       # 获取用户所设置的ACCESS_KEY_ID
@@ -639,6 +636,18 @@ module Aliyun
         @protocol.sign(string_to_sign)
       end
 
+      # Get the download crc status
+      # @return true(download crc enable) or false(download crc disable)
+      def download_crc_enable
+        @protocol.download_crc_enable
+      end
+
+      # Get the upload crc status
+      # @return true(upload crc enable) or false(upload crc disable)
+      def upload_crc_enable
+        @protocol.upload_crc_enable
+      end
+
       private
       # Infer the file's content type using MIME::Types
       # @param file [String] the file path
@@ -655,7 +664,6 @@ module Aliyun
       def get_cpt_file(file)
         "#{File.expand_path(file)}.cpt"
       end
-
     end # Bucket
   end # OSS
 end # Aliyun