fluent-plugin-datahub 0.0.1

data/lib/fluent/plugin/datahub/datahub-http-client.rb

@@ -0,0 +1,229 @@
+require "net/http"
+require "openssl"
+require "base64"
+require "date"
+require "time"
+require "json"
+require_relative "datahub-record-schema"
+require_relative "datahub-record-entity"
+
+########################
+##  @author pholy.ht  ##
+##  @time 2016-03-24  ##
+########################
+class DatahubHttpClient
+    attr_accessor :access_id
+    attr_accessor :access_key
+    attr_accessor :endpoint
+    
+    def initialize(endpoint, access_id, access_key)
+        @endpoint = endpoint
+        @access_id = access_id
+        @access_key = access_key
+    end
+    
+    # Signature = base64(HmacSha1(AccessKeySecret, VERB + "\n"
+    #           + CONTENT-TYPE + "\n"
+    #           + DATE + "\n"
+    #           + CanonicalizedDatahubHeaders + "\n"
+    #           + CanonicalizedResource))
+    
+    def get_signature(params)
+        verb = params["verb"]
+        content_type = params["content-type"]
+        resource = params["resource"]
+        gmt_time = params["date"]
+        data = verb + "\n" + content_type + "\n" + gmt_time + "\n" + "x-datahub-client-version:1\n" + resource
+        # data = "GET\napplication/json\nFri, 06 May 2016 06:43:31 GMT\nx-datahub-client-version:1\n/projects/test_project/topics/datahub_fluentd_out_1"
+        return "DATAHUB " + @access_id + ":" + Base64.encode64("#{OpenSSL::HMAC.digest('sha1', @access_key, data)}").chomp
+    end
+    
+    def send_request(method, path, params={}, headers={})
+        uri = URI(@endpoint + path)
+        http_req = nil
+        if method == "GET"
+            http_req = Net::HTTP::Get.new(uri.path)
+        elsif method == "POST"
+            http_req = Net::HTTP::Post.new(uri.path)
+            # start_time = Time.now.to_f
+            # puts "start_time :" + start_time.to_s
+            http_req.body = JSON.generate(params)
+            # end_time = Time.now.to_f
+            # puts "end_time   :" + end_time.to_s
+            # puts ((end_time - start_time)*1000).to_s
+        elsif method == "PUT"
+            http_req = Net::HTTP::Put.new(uri.path)
+            http_req.body = JSON.generate(params)
+        elsif method == "DELETE"
+            http_req = Net::HTTP::Delete.new(uri.path)
+        else
+            raise "Unsupported method: " + method
+        end
+        
+        sig_params = {}
+        sig_params["verb"] = method
+        sig_params["content-type"] = "application/json" 
+        sig_params["resource"] = path
+        sig_params["date"] = Time.now.utc.strftime("%a, %d %b %Y %T") + " GMT"
+        
+        http_req["Authorization"] = get_signature(sig_params)
+        http_req["x-datahub-client-version"] = 1
+        http_req["Date"] = sig_params["date"]
+        http_req["Content-Type"] = sig_params["content-type"]
+        http_req["ACCEPT-ENCODING"] = nil
+        
+        # puts http_req["Date"]
+        # puts http_req["Authorization"]
+        # puts http_req.to_hash
+        
+        headers.each do |k, v|
+            http_req[k] = v
+        end
+        
+        # puts path
+        
+        return Net::HTTP.start(uri.host, uri.port) {
+            |client|
+            http_resp = client.request(http_req)
+            if http_resp.code != '200' and http_resp.code != '201'
+                if http_resp.body.empty?
+                    raise "Send request failed, unknown response " 
+                end
+                raise http_resp.body
+            end
+            result = http_resp.body
+            if result.empty?
+                return nil
+            end
+            return JSON.parse(result)
+        }
+    end
+    
+    def list_projects()
+        path = "/projects"
+        return send_request("GET", path)
+    end
+    
+    def get_project(project_name)
+        path = "/projects/" + project_name
+        return send_request("GET", path)
+    end
+    
+    def create_project(project_name, comment)
+        path = "/projects/" + project_name
+        params = {}
+        params["Comment"] = comment
+        return send_request("POST", path, params)
+    end
+    
+    # datahub目前未实现update操作
+    # def update_project(project_name, comment)
+    #     path = "/projects/" + project_name
+    #     params = {}
+    #     params["Comment"] = comment
+    #     return send_request("PUT", path, params)
+    # end
+    
+    def delete_project(project_name)
+        path = "/projects/" + project_name
+        return send_request("DELETE", path)
+    end
+    
+    def list_topics(project_name)
+        path = "/projects/" + project_name + "/topics"
+        return send_request("GET", path)
+    end
+    
+    def get_topic(project_name, topic_name)
+        path = "/projects/" + project_name + "/topics/" + topic_name
+        return send_request("GET", path)
+    end
+    
+    def create_topic(project_name, topic_name, shard_count, lifecycle, record_type, record_schema, comment)
+        path = "/projects/" + project_name + "/topics/" + topic_name
+        params = {}
+        params["ShardCount"] = shard_count
+        params["Lifecycle"] = lifecycle
+        params["RecordType"] = record_type
+        params["RecordSchema"] = JSON.generate(record_schema)
+        params["Comment"] = comment
+        return send_request("POST", path, params)
+    end
+    
+    def update_topic(project_name, topic_name, lifecycle, comment)
+        path = "/projects/" + project_name + "/topics/" + topic_name
+        params = {}
+        params["Lifecycle"] = lifecycle
+        params["Comment"] = comment
+        return send_request("PUT", path, params)
+    end
+    
+    def delete_topic(project_name, topic_name)
+        path = "/projects/" + project_name + "/topics/" + topic_name
+        return send_request("DELETE", path)
+    end
+    
+    def list_shards(project_name, topic_name)
+        path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
+        return send_request("GET", path)
+    end
+    
+    ##########################################################
+    ## shard的merge、split未实现
+    ## datahub这一期并没有实现，所以api不开放
+    ##########################################################
+    # def merge_shard(project_name, topic_name, shard_id, adjacent_shard_id)
+    #     path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
+    #     params = {}
+    #     params["Action"] = "merge"
+    #     params["ShardId"] = shard_id
+    #     params["AdjacentShardId"] = adjacent_shard_id
+    #     return send_request("POST", path, params)
+    # end
+    
+    # def split_shard(project_name, topic_name, shard_id, split_key)
+    #     path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
+    #     params = {}
+    #     params["Action"] = "split"
+    #     params["ShardId"] = shard_id
+    #     params["SplitKey"] = split_key
+    #     return send_request("POST", path, params)
+    # end
+    
+    def write_data_to_topic(project_name, topic_name, record_entities)
+        path = "/projects/" + project_name + "/topics/" + topic_name + "/shards"
+        params = {}
+        params["Action"] = "pub"
+        params["Records"] = record_entities
+        return send_request("POST", path, params)
+    end
+    
+    def get_shard_cursor(project_name, topic_name, shard_id, offset=DateTime.now.strftime('%Q'), type="LATEST")
+        path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
+        params = {}
+        params["Action"] = "cursor"
+        params["SystemTime"] = offset
+        params["Type"] = type
+        return send_request("POST", path, params)
+    end
+    
+    def read_data_from_shard_with_cursor(project_name, topic_name, shard_id, cursor, count)
+        path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
+        params = {}
+        params["Action"] = "sub"
+        params["Cursor"] = cursor
+        params["Limit"] = count
+        return send_request("POST", path, params)
+    end
+    
+    def read_data_from_shard(project_name, topic_name, shard_id, count, offset=DateTime.now.strftime('%Q'), type="LATEST")
+        path = "/projects/" + project_name + "/topics/" + topic_name + "/shards/" + shard_id
+        cursor = get_shard_cursor(project_name, topic_name, shard_id, offset, type).to_hash["Cursor"]
+        params = {}
+        params["Action"] = "sub"
+        params["Cursor"] = cursor
+        params["Limit"] = count
+        return send_request("POST", path, params)
+    end
+    
+end

data/lib/fluent/plugin/datahub/datahub-project.rb

@@ -0,0 +1,59 @@
+require_relative "datahub-http-client"
+require_relative "datahub-topic"
+
+class DatahubProject
+    def initialize(datahub_http_client, project_name)
+        @client = datahub_http_client
+        @project_name = project_name
+    end
+
+    def list_topics()
+        topics_map = @client.list_topics
+        topics_array = topics_map["TopicNames"]
+        topics = []
+        for i in 0...topics_array.size
+            topic_name = topics_array[i]
+            topic = DatahubTopic.new(datahub_http_client, @project_name, topic_name)
+            topics.push(topic)
+        end
+        
+        return topics
+    end
+
+    def create_topic(topic_name, shard_count, lifecycle, record_type, record_schema, comment)
+        @client.create_topic(@project_name, topic_name, shard_count, lifecycle, record_type, record_schema, comment)
+        return self.get_topic(topic_name)
+    end
+    
+    def update_topic(topic_name, lifecycle, desc)
+        @client.update_topic(@project_name, topic_name, lifecycle, desc)
+    end
+    
+    def delete_topic(topic_name)
+        @client.delete_topic(@project_name, topic_name)
+    end
+    
+    def get_topic(topic_name)
+        result_map = @client.get_topic(@project_name, topic_name)
+        record_schema_string = result_map["RecordSchema"]
+        record_schema_map = JSON.parse(record_schema_string)
+        fields = record_schema_map["fields"]
+        record_schema = RecordSchema.new()
+        for i in 0...fields.size
+            field = fields[i]
+            record_field = RecordField.new(field["name"], field["type"])
+            record_schema.add_field(record_field)
+        end
+        topic = DatahubTopic.new(@client, @project_name, topic_name)
+        topic.shard_count = result_map["ShardCount"]
+        topic.lifecycle = result_map["Lifecycle"]
+        topic.record_type = result_map["RecordType"]
+        topic.record_schema = record_schema
+        topic.comment = result_map["Comment"]
+        topic.create_time = result_map["CreateTime"]
+        topic.last_modify_time = result_map["LastModifyTime"]
+        
+        return topic
+    end
+    
+end

data/lib/fluent/plugin/datahub/datahub-put-record-result.rb

@@ -0,0 +1,23 @@
+
+class PutRecordResult
+    attr_accessor :failed_record_count
+    attr_accessor :failed_record_index
+    attr_accessor :failed_record_error
+    attr_accessor :failed_record_list
+
+    def initialize
+      @failed_record_count = 0
+      @failed_record_index = []
+      @failed_record_error = []
+      @failed_record_list = []
+    end
+
+    def to_json(*a)
+      {
+          "failed_record_count" => @failed_record_count,
+          "failed_record_index" => @failed_record_index,
+          "failed_record_error" => @failed_record_error,
+          "failed_record_list" => @failed_record_list
+      }.to_json(*a)
+    end
+end

data/lib/fluent/plugin/datahub/datahub-record-entity.rb

@@ -0,0 +1,136 @@
+
+class RecordEntity
+    $STRING_MAX_LENGTH = 1 * 1024 * 1024
+
+    def initialize(schema)
+        @columns_map = {}
+        
+        @shard_id = nil
+        @attributes = {}
+        
+        @schema = schema
+    end
+    
+    def set(name, value)
+        @columns_map[name] = value
+    end
+    
+    def set_shard_id(shard_id)
+        if shard_id.is_a?String
+            @shard_id = shard_id
+        else
+            raise "shard_id must be String type!"
+        end
+    end
+    
+    def add_attribute(name, value)
+        @attributes[name] = value
+    end
+
+    def get_columns_map
+        return @columns_map
+    end
+
+    def get_schema
+        return @schema
+    end
+
+
+    def setBigInt(name, value)
+        if value == nil
+            set(name, value)
+        elsif value.is_a?Integer
+            set(name, value)
+        elsif value.is_a?String and value.to_i.to_s == value
+            set(name, value.to_i)
+        else
+            raise "value show be Integer, name:" + name.to_s + " value:" + value.to_s
+        end
+    end
+
+    def setDouble(name, value)
+        if value == nil
+            set(name, value)
+        elsif value.is_a?Float
+            set(name, value)
+        elsif value.is_a?String
+            begin
+                set(name, Float(value))
+            rescue
+                raise "value show be Float, name:" + name.to_s + " value:" + value.to_s
+            end
+        else
+            raise "value show be Float, name:" + name.to_s + " value:" + value.to_s
+        end
+    end
+
+    def setBoolean(name, value)
+        if value == nil
+            set(name, value)
+        elsif value.is_a?String
+            if value == "true"
+                set(name, true)
+            elsif value == "false"
+                set(name, false)
+            else
+                raise "value must be true or false, name:" + name.to_s + " value:" + value.to_s
+            end
+        elsif value != false and value != true
+            raise "value must be bool or string[true,false], name:" + name.to_s + " value:" + value.to_s
+        end
+        set(name, value)
+    end
+
+    def setTimeStamp(name, value)
+        if value == nil
+            set(name, value)
+        elsif value.is_a?Integer
+            set(name, value)
+        elsif value.is_a?String and value.to_i.to_s == value
+            set(name, value.to_i)
+        else
+            raise "value should be Integer, name:" + name.to_s + " value:" + value.to_s
+        end
+    end
+
+    def setString(name, value)
+        if value == nil
+            set(name, value)
+        elsif value.is_a?String and value.length < $STRING_MAX_LENGTH
+            if @schema.get_encoding == nil
+                set(name, value)
+            else
+                # puts "encoding:" + @schema.get_encoding.to_s
+                value.force_encoding(@schema.get_encoding)
+                set(name, value)
+                # puts value
+            end
+        else
+            raise "value show be String and len < " + $STRING_MAX_LENGTH.to_s + ", name:" + name.to_s + " value:" + value.to_s
+        end
+    end
+    
+    def get_stored_column_values
+        data = []
+        fields = @schema.get_fields
+        for i in 0...fields.size
+            field = fields[i]
+            name = field.get_name
+            if @columns_map[name] == nil
+                data.push(@columns_map[name])
+            elsif
+                data.push(@columns_map[name].to_s)
+            end
+        end
+        return data
+    end
+    
+    def to_json(*a)
+        {
+            "Data" => get_stored_column_values,
+            "ShardId" => @shard_id,
+            "Attributes" => @attributes
+        }.to_json(*a)
+    end
+    
+end