aliyun_sls 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2c73284b6f778826dc960bb7658a8afc0d746575
+  data.tar.gz: 0f9b584ff1d6c1793ac7892691fcca8d5c0e1a80
+SHA512:
+  metadata.gz: b1f74d011a320926289ecd1df1520a09a419ebc3a5b62c7470ba02f746882b497d62e1246c9a14144e5da62e8b1b5446d8909357dbe3efae79aaf0471b458261
+  data.tar.gz: b7345690159e33083a2c8b142bea93cef5c0df254a9db6999f9881a5218bbe826fe12edabc87a4b84a50841b4ad995bb285a3ab77d28bc3d24fe0cb466a444ab

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aliyun_sls.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 cuizheng
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,76 @@
+# AliyunSls
+
+## 阿里云SLS服务Ruby SDK
+
+简单日志服务（Simple Log Service，简称SLS）是针对日志收集、存储和查询的平台化服务。服务提供各种类型日志的实时收集，平台化存储及实时查询海量的日志。并可以将日志归档至ODPS，以利用ODPS做大数据分析。除了通过管理控制台操作，SLS还提供了API（Application Programming Interface）方式写入、查询日志数据，管理自己的项目及日志库等。
+
+#### SLS(简单日志服务)[介绍](http://docs.aliyun.com/?spm=5176.730001.3.10.5GpxDL#/sls)
+
+------------
+## 库用法
+
+### 查询Store清单(ListLogstores)
+
+    con = AliyunSls::Connection.new("project", "region", "access_key_secret", "aliyun_access_key")
+    con.list_logstores
+
+### 上传日志(PutLogs)
+
+    log = AliyunSls::Protobuf::Log.new(:time => Time.now.to_i, :contents => [])
+
+    [
+        ['value1', '12'],
+        ['value2', '24'],
+        ['value3', '36'],
+        ['value4', '48']
+    ].each { |e|  
+        k = e[0]
+        v = e[1]
+        log_item = AliyunSls::Protobuf::Log::Content.new(:key => k, :value => v)
+        log.contents << log_item
+    }
+    log_list = AliyunSls::Protobuf::LogGroup.new(:logs => [])
+    log_list.logs << log
+
+    con = AliyunSls::Connection.new("project", "region", "access_key_secret", "aliyun_access_key")
+    con.puts_logs("store", log_list)
+
+### TODO
+
+- `ListTopics`  列出Logstore中的日志主题。
+- `GetHistograms`   查询Logstore中的日志在时间轴上的分布。
+- `GetLogs` 查询Logstore中的日志数据。
+
+## 命令行用法
+
+1. 生成配置文件
+    2. `PROJECT=[PROJECT] REGION=[REGION] SECRET=[SECRET] KEY=[KEY] sls setup`
+1. 执行上传日志操作
+    2. `[PROJECT=[PROJECT]] sls put log_path store [topic]`
+    2. 上传日志操作是使用Tail方式执行，有增量日志产生的时候，会自动收集增量部分上传到阿里云
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'aliyun_sls'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install aliyun_sls
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/aliyun_sls.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'aliyun_sls/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "aliyun_sls"
+  spec.version       = AliyunSls::VERSION
+  spec.authors       = ["cuizheng"]
+  spec.email         = ["zheng.cuizh@gmail.com"]
+  spec.description   = %q{Gem for SLS of Aliyun}
+  spec.summary       = %q{Gem for SLS of Aliyun}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_dependency "rest-client"
+  spec.add_dependency "beefcake"
+  spec.add_dependency "file-tail"
+  spec.add_dependency "ruby-hmac"
+end

data/bin/sls

@@ -0,0 +1,85 @@
+#!/usr/bin/env ruby
+require File.join(File.dirname(__FILE__), '..', 'lib', 'aliyun_sls.rb')
+require "file-tail"
+require "byebug"
+puts "sls [opt] ..." if (ARGV.size < 1)
+
+def setup
+    h = {}
+    h['project'] = ENV["PROJECT"]
+    h['region'] = ENV["REGION"] || (puts "PROJECT=[PROJECT] REGION=[REGION] SECRET=[SECRET] KEY=[KEY] sls setup";exit)
+    h['access_key_secret'] = ENV["SECRET"] || (puts "PROJECT=[PROJECT] REGION=[REGION] SECRET=[SECRET] KEY=[KEY] sls setup";exit)
+    h['aliyun_access_key'] = ENV["KEY"] || (puts "PROJECT=[PROJECT] REGION=[REGION] SECRET=[SECRET] KEY=[KEY] sls setup";exit)
+    File.open(File.join(Dir.home, '.slsrc'), 'w+'){|f|
+        f << YAML.dump(h)
+    }
+    puts "setup config at ~/.slsrc"
+end
+
+def load
+    File.open(File.join(Dir.home, '.slsrc')){|f|
+        $config = YAML.load(f)
+    }
+end
+
+def regist_exit
+    puts "Press CTRL-C to quit"
+    Signal.trap(:INT){
+        exit(-1)
+    }
+end
+
+def put_logs(path, store, topic)
+    regist_exit
+    (puts "[PROJECT=[PROJECT]] sls put log_path store [topic]";exit(-1)) if !path || !store
+    load
+    (puts "load config failed! run: sls setup first.";exit(-1)) if !$config
+    
+    # 如果传递了项目名称
+    # 则使用本次调用传递过来的值
+    # 否则使用配置文件中的值
+    con = AliyunSls::Connection.new(ENV['PROJECT'] || $config['project'], $config['region'], $config['access_key_secret'], $config['aliyun_access_key'])
+
+    File.open(path) do |log|
+        log.extend(File::Tail)
+        log.interval = 3
+        log.backward(0)
+        log.tail { |line|
+            puts line
+            put_line(line, con, store, topic)
+        }
+    end
+end
+
+def put_line(line, con, store, topic)
+    log = AliyunSls::Protobuf::Log.new(:time => Time.now.to_i, :contents => [])
+    begin
+        uri = URI.parse(line)
+        uri.query and uri.query.split('&').each{|paire| 
+            k, v = paire.split('=')
+            log_item = AliyunSls::Protobuf::Log::Content.new(:key => k, :value => v)
+            log.contents << log_item
+        }
+        log_list = AliyunSls::Protobuf::LogGroup.new(:logs => [])
+        log_list.logs << log
+        log_list.topic = topic
+        log_list.source = uri.host
+    rescue URI::InvalidURIError => e
+        puts e
+        return
+    end
+
+    begin
+        rsp = con.puts_logs(store, log_list)
+    rescue Exception => e
+        puts "Exception with rsp #{e}"
+    end
+end
+
+case opt = ARGV[0]
+when /setup/i
+    setup
+    exit(0)
+when /put/i
+    put_logs(ARGV[1], ARGV[2], ARGV[3] || 'aliyun_sls_topic')
+end

data/lib/aliyun_sls/connection.rb

@@ -0,0 +1,157 @@
+require 'rest_client'
+require 'hmac-sha1'
+require "base64"
+require "zlib"
+
+# log = AliyunSls::Protobuf::Log.new(:time => Time.now.to_i, :contents => [])
+
+# [
+#     ['value1', '12'],
+#     ['value2', '24'],
+#     ['value3', '36'],
+#     ['value4', '48']
+# ].each { |e|  
+#     k = e[0]
+#     v = e[1]
+#     log_item = AliyunSls::Protobuf::Log::Content.new(:key => k, :value => v)
+#     log.contents << log_item
+# }
+# log_list = AliyunSls::Protobuf::LogGroup.new(:logs => [])
+# log_list.logs << log
+
+# con = AliyunSls::Connection.new("project", "region", "access_key_secret", "aliyun_access_key")
+# con.puts_logs("store", log_list)
+
+# con = AliyunSls::Connection.new("project", "region", "access_key_secret", "aliyun_access_key")
+# con.list_logstores
+
+module AliyunSls
+    class Connection
+# region:
+# 杭州：cn-hangzhou.sls.aliyuncs.com
+# 青岛：cn-qingdao.sls.aliyuncs.com
+# 北京：cn-beijing.sls.aliyuncs.com
+# 深圳：cn-shenzhen.sls.aliyuncs.com
+        def initialize(project, region, access_key_secret, aliyun_access_key, opts={})
+            default_headers = {
+                "x-sls-apiversion" => "0.4.0",
+                "x-sls-signaturemethod" => "hmac-sha1"
+            }
+            @headers = default_headers.update opts
+            @aliyun_access_key = aliyun_access_key
+            @access_key_secret = access_key_secret
+            @host = "#{project}.#{region}"
+        end
+
+        def string_to_sign(verb, logstorename, headers, content, query={})
+            if content
+                string_to_sign_with_content(verb, logstorename, headers, query)
+            else
+                string_to_sign_without_content(verb, logstorename, headers, query)
+            end
+        end
+
+        def string_to_sign_with_content(verb, logstorename, headers, query={})
+            <<-DOC
+#{verb}
+#{headers['Content-MD5']}
+#{headers['Content-Type']}
+#{headers['Date']}
+#{canonicalized_sls_headers(headers)}
+#{canonicalized_resource(logstorename, query)}
+DOC
+        end
+
+        def string_to_sign_without_content(verb, logstorename, headers, query={})
+            <<-DOC
+#{verb}
+
+
+#{headers['Date']}
+#{canonicalized_sls_headers(headers)}
+#{canonicalized_resource(logstorename, query)}
+DOC
+        end
+
+# “CanonicalizedSLSHeaders”的构造方式如下：
+# 1. 将所有以“x-sls-”为前缀的HTTP请求头的名字转换成小写字母；
+# 2. 将上一步得到的所有SLS自定义请求头按照字典序进行升序排序；
+# 3. 删除请求头和内容之间分隔符两端出现的任何空格；
+# 4. 将所有的头和内容用\n分隔符组合成最后的CanonicalizedSLSHeader;
+        def canonicalized_sls_headers(headers)
+            h = {}
+            headers.each { |k, v|  
+                if k =~ /x-sls-.*/
+                    h[k.downcase] = v
+                end
+            }
+            h.keys.sort.map { |e|  
+                h[e]
+                "#{e}:#{h[e].gsub(/^\s+/,'')}"
+            }.join($/)
+        end
+
+# “CanonicalizedResource”的构造方式如下：
+# 1. 将CanonicalizedResource设置为空字符串（""）；
+# 2. 放入要访问的SLS资源："/logstores/logstorename"（无logstorename则不填）；
+# 3. 如请求包含查询字符串（QUERY_STRING），则在CanonicalizedResource字符串尾部添加“？”和查询字符串。
+        def canonicalized_resource(logstorename, query={})
+            u = logstorename ? URI.parse("/logstores/#{logstorename}") : URI.parse("/logstores")
+            if query.size != 0
+                u.query = query.inject([]){|s, a| s << "#{a[0]}=#{a[1]}"}.join('&')
+            end
+            u.to_s
+        end
+
+# 目前，SLS API只支持一种数字签名算法，即默认签名算法"hmac-sha1"。其整个签名公式如下：
+# Signature = base64(hmac-sha1(UTF8-Encoding-Of(SignString)，AccessKeySecret))
+        def sign(verb, logstorename, headers, content, query)
+            Base64.encode64((HMAC::SHA1.new(@access_key_secret) << string_to_sign(verb, logstorename, headers, content, query).chomp).digest).strip
+        end
+
+        def signature(verb, logstorename, headers, content, query)
+            "SLS #{@aliyun_access_key}:#{sign(verb, logstorename, headers, content, query)}"
+        end
+
+        # content是LogGroup
+        def compact_headers(content, compressed)
+            headers = @headers.dup
+            # headers["x-sls-date"] = 
+            headers["Date"] = DateTime.now.httpdate
+            headers["x-sls-bodyrawsize"] = "0"
+
+            if content and compressed
+                body = content.encode.to_s
+                headers["Content-Length"] = compressed.bytesize.to_s
+                # 日志内容包含的日志必须小于3MB和4096条。
+                raise AliyunSls::PostBodyTooLarge, "content length is larger than 3MB" if headers["Content-Length"].to_i > 3*1024**2*8
+                raise AliyunSls::PostBodyTooLarge, "content size is more than 4096" if content.logs.size > 4096
+                # MD5必须为大写字符串
+                headers["Content-MD5"] = Digest::MD5.hexdigest(compressed).upcase
+                headers["Content-Type"] = "application/x-protobuf"
+                headers["x-sls-bodyrawsize"] = body.bytesize.to_s
+                headers["x-sls-compresstype"] = "deflate"
+            end
+            headers
+        end
+
+        def puts_logs(logstorename, content)
+            # 压缩content数据
+            compressed = Zlib::Deflate.deflate(content.encode.to_s)
+            headers = compact_headers(content, compressed)
+            headers["Authorization"] = signature("POST", logstorename, headers, content, {})
+
+            u = URI.parse("http://#{@host}/logstores/#{logstorename}")
+            RestClient.post u.to_s, compressed, headers
+        end
+
+        def list_logstores
+            headers = compact_headers(nil, nil)
+            headers["Authorization"] = signature("GET", nil, headers, nil, {})
+
+            u = URI.parse("http://#{@host}/logstores")
+            headers["Referer"] = u.to_s
+            RestClient.get u.to_s, headers
+        end
+    end
+end

data/lib/aliyun_sls/protobuf.rb

@@ -0,0 +1,50 @@
+require 'beefcake'
+
+module AliyunSls
+    module Protobuf
+
+# message Log
+# {
+#     required uint32 time = 1; // UNIX Time Format
+#     message Content
+#     {
+#         required string key = 1;
+#         required string value = 2;
+#     }
+#     repeated Content contents= 2;
+# }
+
+# message LogGroup
+# {
+#     repeated Log logs= 1;
+#     optional string reserved =2; // 内部字段，不需要填写
+#     optional string topic = 3;
+#     optional string source = 4;
+# }
+
+        class Log
+            include Beefcake::Message
+
+            required :time, :uint32, 1
+
+            class Content
+                include Beefcake::Message
+
+                required :key, :string, 1
+                required :value, :string, 2
+            end
+
+            repeated :contents, Content, 2
+        end
+
+        class LogGroup
+            include Beefcake::Message
+
+            repeated :logs, Log, 1
+            optional :reserved, :string, 2
+            optional :topic, :string, 3
+            optional :source, :string, 4
+        end
+        
+    end
+end

data/lib/aliyun_sls/version.rb

@@ -0,0 +1,3 @@
+module AliyunSls
+  VERSION = "0.0.2"
+end

data/lib/aliyun_sls.rb

@@ -0,0 +1,13 @@
+require File.join(File.dirname(__FILE__), 'aliyun_sls', 'version.rb')
+require File.join(File.dirname(__FILE__), 'aliyun_sls', 'protobuf.rb')
+require File.join(File.dirname(__FILE__), 'aliyun_sls', 'connection.rb')
+
+module AliyunSls
+    class PostBodyInvalid < RuntimeError; end
+    class SLSInvalidTimestamp < RuntimeError; end
+    class SLSInvalidEncoding < RuntimeError; end
+    class SLSInvalidKey < RuntimeError; end
+    class PostBodyTooLarge < RuntimeError; end
+    class PostBodyUncompressError < RuntimeError; end
+    class SLSLogStoreNotExist < RuntimeError; end
+end

metadata

@@ -0,0 +1,141 @@
+--- !ruby/object:Gem::Specification
+name: aliyun_sls
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- cuizheng
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-08 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rest-client
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: beefcake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: file-tail
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: ruby-hmac
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Gem for SLS of Aliyun
+email:
+- zheng.cuizh@gmail.com
+executables:
+- sls
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- aliyun_sls.gemspec
+- bin/sls
+- lib/aliyun_sls.rb
+- lib/aliyun_sls/connection.rb
+- lib/aliyun_sls/protobuf.rb
+- lib/aliyun_sls/version.rb
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.6
+signing_key: 
+specification_version: 4
+summary: Gem for SLS of Aliyun
+test_files: []
+has_rdoc: