fluent-plugin-datahub 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 922859012ad1c685d8396cca953cc6ca35471758
+  data.tar.gz: 646a600f82a8891f8fcb8e05de4db5ccbeecaadc
+SHA512:
+  metadata.gz: 9f36f93349cb259c9c9194909186ef5d3322dabe0a1287698ec04f803e9d0c971552bb9db91ae9d6bd941e8f506990c7c9d31ec84e750f31e6c29df69241323f
+  data.tar.gz: 861bf12a2478ee7458d50f2843295f59be030e3410f8c1fd9f463681fffce8ebea340e834c8af8045db3d63e312e5eaae3c50d7afdb090e4ab9bb7a64aef67d5

data/Gemfile

@@ -0,0 +1,3 @@
+source "http://rubygems.org"
+
+gemspec

data/README.md

@@ -0,0 +1,38 @@
+# fluent-plugin-datahub
+ 概述  
+-------
+  fluent的datahub上传插件
+安装 
+-------
+gem install fluent-plugin-datahub
+
+配置
+-------
+```
+<source>
+  @type tail
+  path ${DIR}/csv_sample.csv
+  tag test1
+  format csv
+  keys id,name,gender,salary,my_time
+</source>
+
+```
+<match test1>
+  @type datahub
+  access_id 
+  access_key 
+  endpoint 
+  project_name test_project
+  topic_name fluentd_out_7
+  # shard_id 6
+  column_names ["id", "name", "gender", "salary", "my_time"]
+  flush_interval 10s
+  dirty_data_continue true
+  dirty_data_file ${DIR}/dirty.data
+  retry_times 3
+</match>
+
+ ```
+  1、source标签中的keys为源数据，会根据key对应fields中字段\<br>
+  2、match标签中的column_names为要写入datahub的字段

data/Rakefile

@@ -0,0 +1,14 @@
+
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test_*.rb']
+  test.verbose = true
+end
+
+task :default => [:build]
+

data/VERSION

@@ -0,0 +1 @@
+0.0.1

data/build.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+mkdir package
+mkdir package/temp
+mkdir package/temp/fluentd-with-datahub
+gem build fluent-plugin-datahub.gemspec
+cp -r ext/* ./package/temp/fluentd-with-datahub/
+cp README.md ./package/temp/fluentd-with-datahub/README.md
+cp -r sample ./package/temp/fluentd-with-datahub/
+cp fluent-plugin-datahub-*.gem ./package/temp/fluentd-with-datahub/dependency_gem/
+tar zcvf ./package/fluentd-with-datahub-0.12.23.tar.gz -C ./package/temp/ .
+rm -rf ./package/temp/

data/fluent-plugin-datahub.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path('../lib', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-datahub"
+  gem.description = "Aliyun Datahub output plugin for Fluentd event collector"
+  gem.license     = "Apache-2.0"
+  gem.homepage    = "https://datahub.console.aliyun.com/datahub"
+  gem.summary     = gem.description
+  gem.version     = File.read("VERSION").strip
+  gem.authors     = ["Aliyun"]
+  gem.email       = "stream@service.aliyun.com"
+  gem.has_rdoc    = false
+  #gem.platform    = Gem::Platform::RUBY
+  gem.files       = `git ls-files | grep -v ext | grep -v package`.split("\n")
+  gem.test_files  = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ['lib']
+
+  gem.add_dependency "fluentd", [">= 0.12.0", "< 2"]
+  gem.add_dependency "yajl-ruby", "~> 1.0"
+end

data/lib/fluent/plugin/datahub/datahub-client.rb

@@ -0,0 +1,27 @@
+require_relative "datahub-http-client"
+require_relative "datahub-project"
+require_relative "datahub-topic"
+
+class DatahubClient
+
+    def initialize(endpoint, access_id, access_key)
+        @endpoint = endpoint
+        @access_id = access_id
+        @access_key = access_key
+        
+        @http_client = DatahubHttpClient.new(@endpoint, @access_id, @access_key)
+    end
+
+    def create_project(project_name, comment)
+        @http_client.create_project(project_name, comment)
+    end
+    
+    def delete_project(project_name)
+        @http_client.delete_project(project_name)
+    end
+    
+    def get_project(project_name)
+        return DatahubProject.new(@http_client, project_name)        
+    end
+
+end

data/lib/fluent/plugin/datahub/datahub-http-client-test.rb

@@ -0,0 +1,343 @@
+require "test/unit"
+require_relative "datahub-http-client"
+require_relative "datahub-client"
+require_relative "datahub-record-schema"
+require_relative "datahub-record-entity"
+
+class DatahubHttpClientTest < Test::Unit::TestCase
+    # def setup
+        # @client = DatahubHttpClient.new("http://10.101.200.231:12357", "63wd3dpztlmb5ocdkj94pxmm", "oRd30z7sV4hBX9aYtJgii5qnyhg=")
+
+        # @project_name = "ruby_sdk_test_project"
+        # begin
+        #     @client.delete_project(@project_name)
+        # rescue
+        # end
+        #
+        # @comment = "fuju test project"
+        # @client.create_project(@project_name, @comment)
+        #
+        # @topic_name = "ruby_sdk_test_table_01"
+        #
+        # @record_schema = RecordSchema.new
+        # @record_schema.add_field(RecordField.new("id","BIGINT"))
+        # @record_schema.add_field(RecordField.new("name","STRING"))
+        # @record_schema.add_field(RecordField.new("gender","BOOLEAN"))
+        # @record_schema.add_field(RecordField.new("salary","DOUBLE"))
+        # @record_schema.add_field(RecordField.new("gmt_create","TIMESTAMP"))
+        #
+        # @topic_comment = "test table"
+        #
+        # begin
+        #     @client.delete_topic(@project_name, @topic_name)
+        # rescue
+        # end
+        #
+        # @client.create_topic(@project_name, @topic_name, 10, 7, "TUPLE", @record_schema, @topic_comment)
+    # end
+
+    # def teardown
+    #     # @client.delete_project(@project_name)
+    #     # @client.delete_topic(@project_name, @topic_name)
+    # end
+    #
+    # def test_list_projects
+    #     puts @client.list_projects
+    # end
+    #
+    # def test_get_project
+    #     project_name = "test_project_5"
+    #     @comment = "test"
+    #     @client.create_project(project_name, @comment)
+    #     # project存在
+    #     project_map = @client.get_project(project_name)
+    #     puts project_map
+    #     assert_equal(project_map["Comment"], @comment)
+    #     @client.delete_project(project_name)
+    #
+    #     # project不存在
+    #     begin
+    #         project_map = @client.get_project("not_exists_project_name")
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "NoSuchProject")
+    #     end
+    # end
+    #
+    # def test_create_project
+    #     test_project_name = "test_project_3"
+    #     begin
+    #         @client.delete_project(test_project_name)
+    #     rescue => e
+    #     end
+    #
+    #     comment = "jype"
+    #     @client.create_project(test_project_name, comment)
+    #     project = @client.get_project(test_project_name)
+    #
+    #     assert_equal(project["Comment"], comment)
+    #
+    #     # project重复
+    #     begin
+    #         @client.create_project(test_project_name, comment)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "ProjectAlreadyExist")
+    #     end
+    #
+    #     # 清理
+    #     @client.delete_project(test_project_name)
+    # end
+    #
+    #
+    # def test_delete_project
+    #     test_project_name = "jyp_production_test_create_project_del"
+    #     comment = "jype"
+    #     @client.create_project(test_project_name, comment)
+    #     project = @client.get_project(test_project_name)
+    #     assert(project != nil, "project should be not null")
+    #     @client.delete_project(test_project_name)
+    #     begin
+    #         new_project = @client.get_project(test_project_name)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "NoSuchProject")
+    #     end
+    # end
+    #
+    # def test_list_topics
+    #     puts @client.list_topics(@project_name)
+    # end
+    #
+    # def test_write_and_read
+    #     @client = DatahubClient.new("http://10.101.200.231:12357", "63wd3dpztlmb5ocdkj94pxmm", "oRd30z7sV4hBX9aYtJgii5qnyhg=")
+    #     @datahub_project = @client.get_project("test_project")
+    #     # @datahub_topic = @datahub_project.get_topic("fluentd_out_6")
+    #     # abc = @datahub_topic.record_schema
+    #     # fields = abc.get_fields
+    #     # puts fields[0].get_name
+    #
+    #     record_schema = RecordSchema.new
+    #     record_schema.add_field(RecordField.new("id","BIGINT"))
+    #     record_schema.add_field(RecordField.new("name","STRING"))
+    #     record_schema.add_field(RecordField.new("gender","BOOLEAN"))
+    #     record_schema.add_field(RecordField.new("salary","DOUBLE"))
+    #     record_schema.add_field(RecordField.new("gmt_create","TIMESTAMP"))
+    #
+    #     #@datahub_project.create_topic("test_ruby_sdk_table_70", 12, 5, "TUPLE", record_schema, "test for ruby sdk")
+    #     my_topic = @datahub_project.get_topic("test_ruby_sdk_table_70")
+    #     # assert_equal(my_topic.record_schema.get_fields[0].get_name, "id")
+    #
+    #     record_entities = []
+    #     record_entity = RecordEntity.new(record_schema)
+    #     record_entity.setBigInt("id", 1)
+    #     record_entity.setString("name", "hj")
+    #     record_entity.set_shard_id("0")
+    #     record_entities.push(record_entity)
+    #     my_topic.write_data(record_entities)
+    #
+    #     cursor = my_topic.get_cursor("0")
+    #     result = my_topic.read_data("0", cursor, 2)
+    #     puts result
+    #
+    # end
+    #
+    #
+    # def hash_code(str)
+    #     str.each_char.reduce(0) do |result, char|
+    #         [((result << 5) - result) + char.ord].pack('L').unpack('l').first
+    #     end
+    # end
+    #
+    # def test_hash
+    #     string1 = "abcdefg,makeboluogggg"
+    #
+    #     puts hash_code(string1)
+    # end
+    #
+    # def test_get_topic
+    #     actual_topic = @client.get_topic(@project_name, @topic_name)
+    #     puts actual_topic
+    #     assert_equal(actual_topic["Comment"], @topic_comment)
+    #     assert_equal(actual_topic["RecordType"], "TUPLE")
+    #     assert_equal(actual_topic["ShardCount"], 10)
+    #
+    # end
+    #
+    # def test_create_topic
+    #     record_schema = RecordSchema.new()
+    #     id = RecordField.new("id", "BIGINT")
+    #     name = RecordField.new("name", "STRING")
+    #     age = RecordField.new("age", "BIGINT")
+    #
+    #     record_schema.add_field(id)
+    #     record_schema.add_field(name)
+    #     record_schema.add_field(age)
+    #
+    #     topic_comment = "jyp_production_made_in_asia_test_topic_create_topic"
+    #     topic_name = "jyp_production_made_in_asia_test_topic_create_topic"
+    #
+    #     @client.create_topic(@project_name, topic_name, 4, 7, "TUPLE", record_schema, topic_comment)
+    #     actual_topic = @client.get_topic(@project_name, topic_name)
+    #     assert_equal(actual_topic["Comment"], topic_comment)
+    #     assert_equal(actual_topic["RecordType"], "TUPLE")
+    #     assert_equal(actual_topic["ShardCount"], 4)
+    #
+    #     # 重复创建
+    #     begin
+    #         @client.create_topic(@project_name, topic_name, 4, 7, "TUPLE", record_schema, topic_comment)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "TopicAlreadyExist")
+    #     end
+    #
+    #     @client.delete_topic(@project_name, topic_name)
+    #
+    #     # 工程不存在
+    #     begin
+    #         @client.create_topic("project_not_exists_aa", topic_name, 4, 7, "TUPLE", record_schema, topic_comment)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "NoSuchProject")
+    #     end
+    #
+    #     # 无效的参数
+    #     begin
+    #         @client.create_topic(@project_name, topic_name, 4, nil, "TUPLE", record_schema, topic_comment)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "InvalidParameter")
+    #     end
+    # end
+    #
+    # def test_update_topic
+    #     record_schema = RecordSchema.new()
+    #     id = RecordField.new("id", "BIGINT")
+    #     name = RecordField.new("name", "STRING")
+    #     age = RecordField.new("age", "BIGINT")
+    #
+    #     record_schema.add_field(id)
+    #     record_schema.add_field(name)
+    #     record_schema.add_field(age)
+    #
+    #     topic_comment = "jyp_production_made_in_asia_test_topic_update_topic"
+    #     topic_new_comment = "jyp_production_made_in_asia_test_topic_update_topic_new"
+    #     topic_name = "jyp_production_made_in_asia_test_topic_update_topic"
+    #
+    #     @client.create_topic(@project_name, topic_name, 4, 7, "TUPLE", record_schema, topic_comment)
+    #     actual_topic = @client.get_topic(@project_name, topic_name)
+    #     assert_equal(actual_topic["Comment"], topic_comment)
+    #
+    #     @client.update_topic(@project_name, topic_name, 6, topic_new_comment)
+    #     actual_topic = @client.get_topic(@project_name, topic_name)
+    #     assert_equal(actual_topic["Comment"], topic_new_comment)
+    #     assert_equal(actual_topic["Lifecycle"], 6)
+    #
+    #     @client.delete_topic(@project_name, topic_name)
+    #
+    # end
+    #
+    # def test_delete_topic
+    #     record_schema = RecordSchema.new()
+    #     id = RecordField.new("id", "BIGINT")
+    #     name = RecordField.new("name", "STRING")
+    #     age = RecordField.new("age", "BIGINT")
+    #
+    #     record_schema.add_field(id)
+    #     record_schema.add_field(name)
+    #     record_schema.add_field(age)
+    #
+    #     topic_comment = "jyp_production_made_in_asia_test_topic_delete_topic"
+    #     topic_name = "jyp_production_made_in_asia_test_topic_delete_topic"
+    #
+    #     @client.create_topic(@project_name, topic_name, 4, 7, "TUPLE", record_schema, topic_comment)
+    #     actual_topic = @client.get_topic(@project_name, topic_name)
+    #     assert(actual_topic != nil, "topic can not be null")
+    #     @client.delete_topic(@project_name, topic_name)
+    #
+    #     begin
+    #         actual_topic = @client.get_topic(@project_name, topic_name)
+    #     rescue => e
+    #         resp = JSON.parse(e.message)
+    #         assert_equal(resp["ErrorCode"], "NoSuchTopic")
+    #     end
+    # end
+    #
+    # def test_list_shard
+    #     puts @client.list_shards(@project_name, @topic_name)
+    # end
+    # #
+    # def test_write_data_to_topic
+    #     sleep 1
+    #     entity = RecordEntity.new(@record_schema)
+    #     entity.set("id", 123)
+    #     entity.set("name", "jyp")
+    #     entity.set("age", 24)
+    #     entity.set_shard_id("0")
+    #     record_entities = []
+    #     record_entities.push(entity)
+    #     puts @client.write_data_to_topic(@project_name, @topic_name, record_entities)
+    # end
+    #
+    # def test_get_shard_cursor
+    #     sleep 1
+    #     puts @client.get_shard_cursor(@project_name, @topic_name, "0", 10)
+    # end
+    #
+    # def test_read_data_from_shard
+    #     sleep 1
+    #     record_entities = []
+    #     # 写入0号shard
+    #     entity1 = RecordEntity.new(@record_schema)
+    #     entity1.set("id", 1111)
+    #     entity1.set("name", "jyp111")
+    #     entity1.set("age", 24)
+    #     entity1.set_shard_id("0")
+    #     record_entities.push(entity1)
+    #
+    #     # 写入1号shard
+    #     entity2 = RecordEntity.new(@record_schema)
+    #     entity2.set("id", 2222)
+    #     entity2.set("name", "jyp222")
+    #     entity2.set("age", 25)
+    #     entity2.set_shard_id("1")
+    #     record_entities.push(entity2)
+    #
+    #     puts @client.write_data_to_topic(@project_name, @topic_name, record_entities)
+    #
+    #     sleep 1
+    #     # 消费0、1shard的数据
+    #     puts @client.read_data_from_shard(@project_name, @topic_name, "0", 10, DateTime.now.strftime('%Q'), "OLDEST")
+    #     puts @client.read_data_from_shard(@project_name, @topic_name, "1", 10, DateTime.now.strftime('%Q'), "OLDEST")
+    # end
+
+    def test
+        @client = DatahubHttpClient.new("http://10.101.200.231:12357", "63wd3dpztlmb5ocdkj94pxmm", "oRd30z7sV4hBX9aYtJgii5qnyhg=")
+
+        record_schema = RecordSchema.new
+        record_schema.setEncoding("UTF-8")
+        record_schema.add_field(RecordField.new("id","BIGINT"))
+        record_schema.add_field(RecordField.new("name","STRING"))
+        record_schema.add_field(RecordField.new("gender","BOOLEAN"))
+        record_schema.add_field(RecordField.new("salary","DOUBLE"))
+        record_schema.add_field(RecordField.new("gmt_create","TIMESTAMP"))
+        record_entities = []
+        record_entity = RecordEntity.new(record_schema)
+        record_entity.setBigInt("id", 1)
+        tmp = "test哈哈中国人"
+        #p tmp.encoding
+        #tmp.force_encoding("ASCII-8BIT")
+        #p tmp.encoding
+
+
+        record_entity.setString("name",tmp)
+        record_entity.set_shard_id("1")
+        record_entity.setDouble("salary","10.00")
+        record_entities.push(record_entity)
+
+        @client.write_data_to_topic("test_project", "datahub_file_result", record_entities)
+
+        puts "ok"
+    end
+    
+end