logstash-output-datahub 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 41b554cbbdc8d2ea64dd05f8284660a499ce1346
+  data.tar.gz: df5877d7d4039a857c1fcaa2e5ba195ca7d0e7e7
+SHA512:
+  metadata.gz: c7fa8a628c8953db4af20fc7938879884484d092248c3e6bbbf476f785bec4509b305c55428f4b25b2603e572a10013143a35feb215248b17f4d8232a285fdf6
+  data.tar.gz: 59a9bb7aea19acea1f829b1891f9702141368764d2351cada3ecdb9e7471f1df82340ba5ba41f97b84ad31a36ff182479d753bff02bd41b4fcf1b6648824c99f

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+0.0.1
+First version.

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,115 @@
+# Aliyun DataHub Plugin for LogStash
+
+## Getting Started
+---
+
+### 介绍
+
+- 该插件是基于logstash开发的输出插件，它主要完成数据采集到DataHub(阿里云产品)存储服务上
+
+### 安装
++ 环境要求linux, jdk1.7+, logstash（可选，如果没安装也没关系）
++ 从流计算官网下载tar包，使用以下命令安装
+
+如果之前没安装过logstash，请使用以下步骤安装
+
+```
+$ tar -xzvf logstash-with-datahub-2.3.0.tar.gz
+$ cd logstash-with-datahub-2.3.0
+```
+
+如果之前安装过logstash，拿到logstash-output-datahub-1.0.0.gem，再使用以下命令安装
+
+```
+$ ${LOGSTASH_HOME}/bin/logstash-plugin install --local logstash-output-datahub-1.0.0.gem
+```
+
+### 样例
++ 采集日志
+
+以下是一个应用打出的日志，格式如下：
+
+```
+20:04:30.359 [qtp1453606810-20] INFO  AuditInterceptor - [13pn9kdr5tl84stzkmaa8vmg] end /web/v1/project/fhp4clxfbu0w3ym2n7ee6ynh/statistics?executionName=bayes_poc_test GET, 187 ms
+```
+
+在datahub的topic的schema如下：
+
+```
+request_time, STRING
+thread_id, STRING
+log_level, STRING
+class_name, STRING
+request_id, STRING
+detail, STRING
+```
+
+logstash的配置如下：
+
+```
+input {
+    file {
+		path => "${APP_HOME}/log/app.log"
+		start_position => "beginning"
+    }
+}
+
+filter{
+    grok {
+		match => {
+		   "message" => "(?<request_time>\d\d:\d\d:\d\d\.\d+)\s+\[(?<thread_id>[\w\-]+)\]\s+(?<log_level>\w+)\s+(?<class_name>\w+)\s+\-(?<detail>.+)"
+		}
+    }
+}
+
+output {
+    datahub {
+		access_id => ""
+		access_key => ""
+		endpoint => ""
+		project_name => ""
+		topic_name => ""
+		#shard_id => "0"
+		#shard_keys => ["thread_id"]
+		batch_size => 10
+		batch_timeout => 5
+		dirty_data_continue => true
+		dirty_data_file => "/Users/ph0ly/trash/dirty.data"
+		dirty_data_file_max_size => 1000
+    }
+}
+```
+
+### 参数介绍
+```
+access_id(Required): 阿里云access id
+access_key(Required): 阿里云access key
+endpoint(Required): 阿里云datahub的服务地址
+project_name(Required): datahub项目名称
+topic_name(Required): datahub topic名称
+retry_times(Optional): 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
+retry_interval(Optional): 下一次重试的间隔，单位为秒
+batch_size(Optional): 批量提交大小，指定数据积攒到@batch_size大小时触发一次提交，默认100
+batch_timeout(Optional): 批量提交超时，在数据量较少的情况下，数据超时后的超时提交，默认5秒
+shard_keys(Optional)：数组类型，数据落shard的字段名称，插件会根据这些字段的值计算hash将每条数据落某个shard, 注意shard_keys和shard_id都未指定，默认轮询落shard
+shard_id(Optional): 所有数据落指定的shard，注意shard_keys和shard_id都未指定，默认轮询落shard
+dirty_data_continue(Optional): 脏数据是否继续运行，默认为false，如果指定true，则遇到脏数据直接无视，继续处理数据。当开启该开关，必须指定@dirty_data_file文件
+dirty_data_file(Optional): 脏数据文件名称，当数据文件名称，在@dirty_data_continue开启的情况下，需要指定该值。特别注意：脏数据文件将被分割成两个部分.part1和.part2，part1作为更早的脏数据，part2作为更新的数据
+dirty_data_file_max_size(Optional): 脏数据文件的最大大小，该值保证脏数据文件最大大小不超过这个值，目前该值仅是一个参考值
+```
+
+## 相关参考
+---
+
+- [LogStash主页](https://www.elastic.co/products/logstash)
+- [LogStash插件开发](https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_output_plugin.html#_coding_output_plugins)
+
+## Authors && Contributors
+---
+
+- [Huang Tao](https://github.com/ph0ly)
+
+## License
+---
+
+licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)

data/lib/logstash/outputs/datahub-test.rb

@@ -0,0 +1,11 @@
+require "test/unit"
+require "mocha"
+require "datahub"
+
+class DatahubTest < Test::Unit::TestCase
+
+    def test_register()
+        
+    end
+    
+end

data/lib/logstash/outputs/datahub.rb

@@ -0,0 +1,341 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/environment"
+require "fileutils"
+require "thread"
+
+jar_path=File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
+LogStash::Environment.load_runtime_jars! File.join(jar_path, "vendor")
+
+# Datahub output plugin
+class LogStash::Outputs::Datahub < LogStash::Outputs::Base
+  declare_threadsafe!
+
+  config_name "datahub"
+
+  # datahub access id
+  config :access_id, :validate => :string, :required => true
+
+  # datahub access key
+  config :access_key, :validate => :string, :required => true
+
+  # datahub service endpoint
+  config :endpoint, :validate => :string, :required => true
+
+  # datahub project name
+  config :project_name, :validate => :string, :required => true
+
+  # datahub topic name
+  config :topic_name, :validate => :string, :required => true
+
+  # 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
+  config :retry_times, :validate => :number, :required => false, :default => -1
+
+  # 重试周期，下一次重试的间隔，单位为秒
+  config :retry_interval, :validate => :number, :required => false, :default => 5
+
+  # 按照指定字段的值计算hash，依据于该hash值落某个shard
+  config :shard_keys, :validate => :array, :required => false, :default => []
+
+  # 指定数据落指定的shard
+  config :shard_id, :validate => :string, :required => false, :default => ""
+
+  #   # 提交的列名，用户可以配置topic的列，采集部分列或者全部列
+  #   # 默认为空数组，表示按照topic的顺序及全字段提交
+  #   # 另外：列的配置不用保序，但是要求该字段在topic的schema中存在
+  #   config :column_names, :validate => :array, :required => false, :default => []
+
+  # 当出现脏数据时，是否继续写入
+  # 当开启该开关，必须指定@dirty_data_file文件
+  config :dirty_data_continue, :validate => :boolean, :required => false, :default => false
+
+  # 脏数据文件名称，当数据文件名称，在@dirty_data_continue开启的情况下，需要指定该值
+  # 特别注意：脏数据文件将被分割成两个部分.part1和.part2，part1作为更早的脏数据，part2作为更新的数据
+  config :dirty_data_file, :validate => :string, :required => false
+
+  # 脏数据文件的最大大小，该值保证脏数据文件最大大小不超过这个值，目前该值仅是一个参考值
+  config :dirty_data_file_max_size, :validate => :number, :required => false, :default => 50024000
+
+  # 数据传输压缩方式选择，目前支持deflate, lz4格式 
+  config :compress_method, :validate => :string, :required => false, :default => ""
+
+  # 该值内部使用，不提供配置
+  # 分发shard的游标
+  attr_accessor :shard_cursor
+
+  # Shard cursor lock
+  @@shard_lock = Mutex.new
+
+  # 写文件锁
+  @@file_lock = Mutex.new
+
+  DatahubPackage = com.aliyun.datahub
+
+  public
+  def register
+    begin
+      @account = DatahubPackage.auth.AliyunAccount::new(@access_id, @access_key)
+      @conf = DatahubPackage.DatahubConfiguration::new(@account, @endpoint)
+      if @compress_method == "deflate" || @compress_method == "lz4"
+        @compression_format = DatahubPackage.model.compress.CompressionFormat.fromValue(@compress_method)
+        @conf.setCompressionFormat(@compression_format)
+      end
+
+      @client = DatahubPackage.DatahubClient::new(@conf)
+      @project = DatahubPackage.wrapper.Project::Builder.build(@project_name, @client)
+      @topic = @project.getTopic(@topic_name)
+      @shard_cursor = 0
+
+      @shards = get_active_shards(@topic.listShard())
+      @shard_count = @shards.size()
+
+      result = @client.getTopic(@project_name, @topic_name)
+      @schema = result.getRecordSchema()
+      fields = @schema.getFields()
+      @columns_size = fields.size
+      @columns = []
+      for i in 0...@columns_size
+        @columns.push(fields[i].getName())
+      end
+
+      # 前置校验参数
+      check_params()
+
+      if @shard_count == 0
+        @logger.error "No active shard available, please check"
+        raise "No active shard available, please check"
+      end
+
+      @logger.info "Init datahub success!"
+    rescue => e
+      @logger.error "Init failed!"  + e.message + " " + e.backtrace.inspect.to_s
+      raise e
+    end
+  end # def register
+
+  def check_params()
+    # 如果shard_id配置了，则检查该shard是否ok
+    if !@shard_id.empty?
+      valid = false
+      for i in 0...@shards.size
+        shard_entry = @shards[i]
+        if shard_entry.getShardId() == @shard_id && shard_entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+          valid = true
+        end
+      end
+      if (!valid)
+        @logger.error "Config shard_id not exists or state not active, check your config"
+        raise "Config shard_id not exists or state not active, check your config"
+      end
+    end
+
+    # 检查shard_keys字段是否合法
+    if @shard_keys.size > 0
+      for i in 0...@shard_keys.size
+        shard_key = @shard_keys[i]
+        if !@schema.containsField(shard_key)
+          @logger.error "Config shard_keys contains one or one more unknown field, check your config"
+          raise "Config shard_keys contains one or one more unknown field, check your config"
+        end
+      end
+    end
+
+    # 配置了脏数据继续，必须指定脏数据文件
+    if @dirty_data_continue
+      if @dirty_data_file.to_s.chomp.length == 0
+        raise "Dirty data file path can not be empty"
+      end
+    end
+
+  end
+
+  # 检查并设置数据到entry中
+  # 如果解析数据异常，则数据落脏数据文件
+  def check_and_set_data(entry, field_type, index, event_map, column_name)
+    data = event_map[column_name]
+    begin
+      if field_type == DatahubPackage.common.data.FieldType::STRING
+        entry.setString(index, data.to_s)
+      elsif field_type == DatahubPackage.common.data.FieldType::BIGINT
+        entry.setBigint(index, java.lang.Long.parseLong(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::DOUBLE
+        entry.setDouble(index, java.lang.Double.parseDouble(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::BOOLEAN
+        entry.setBoolean(index, java.lang.Boolean.parseBoolean(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::TIMESTAMP
+        entry.setTimeStamp(index, java.lang.Long.parseLong(data.to_s))
+      else
+        raise "Unknown schema type of data"
+      end
+      return true
+    rescue => e
+      @logger.error "Parse data: " + column_name + "[" + data + "] failed, " + e.message
+      # 数据格式有异常，根据配置参数确定是否续跑
+      if !@dirty_data_continue
+        @logger.error "Dirty data found, exit process now."
+        puts "Dirty data found, exit process now."
+        Process.exit(1)
+        # 忽略的异常数据直接落文件
+      else
+        write_as_dirty_data(event_map)
+      end
+      return false
+    end
+  end
+
+  # 脏数据文件处理
+  def write_as_dirty_data(event_amp)
+    dirty_file_part1_name = @dirty_data_file + ".part1"
+    dirty_file_part2_name = @dirty_data_file + ".part2"
+
+    # 加锁写入
+    @@file_lock.synchronize {
+      dirty_file_part2 = File.open(dirty_file_part2_name, "a+")
+      dirty_file_part2.puts(event_amp.to_s)
+      dirty_file_part2.close
+      if File.size(dirty_file_part2_name) > @dirty_data_file_max_size / 2
+        # .part1, .part2分别存储数据
+        # 旧数据落part1，新的数据落part2
+        FileUtils.mv(dirty_file_part2_name, dirty_file_part1_name)
+      end
+    }
+  end
+
+  def get_active_shards(shards)
+    active_shards = []
+    for i in 0...shards.size
+      entry = shards.get(i)
+      if entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+        active_shards.push(entry)
+      end
+    end
+    return active_shards
+  end
+
+  def get_next_shard_id()
+    if !@shard_id.empty?
+      return @shard_id
+      # 否则轮询写入shard
+    else
+      idx = 0
+      @@shard_lock.synchronize {
+        idx = @shard_cursor % @shard_count
+        @shard_cursor = idx + 1
+      }
+      shard_id = @shards[idx].getShardId()
+      return shard_id
+    end
+  end
+
+  def multi_receive(event_list)
+    begin
+      entries = []
+      shard_id = get_next_shard_id()
+
+      event_list.each do |event|
+        if event == LogStash::SHUTDOWN
+          return
+        end
+        event_map = event.to_hash
+
+        entry = DatahubPackage.model.RecordEntry::new(@schema)
+        #entry.putAttribute("srcId", event_map["host"].to_s)
+        #entry.putAttribute("ts", event_map["@timestamp"].to_s)
+        #entry.putAttribute("version", event_map["@version"].to_s)
+        #entry.putAttribute("srcType", "log")
+
+        for i in 0...@columns_size do
+          value = event_map[@columns[i]]
+          if value != nil
+            entry.set(i, value)
+          end
+        end
+
+        if @shard_keys.size > 0
+          hash_string = ""
+          for i in 0...@shard_keys.size
+            shard_key = @shard_keys[i]
+            if event_map[shard_key] != nil
+              hash_string += event_map[shard_key].to_s + ","
+            end
+          end
+          hashed_value = java.lang.String.new(hash_string).hashCode()
+          entry.setPartitionKey(hashed_value)
+        else
+          entry.setShardId(shard_id)
+        end
+        entries.push(entry)
+      end
+
+      # puts "total: " + entries.size.to_s
+
+      # 提交列表必须有数据
+      if entries.size > 0
+        put_result = @client.putRecords(@project_name, @topic_name, entries)
+        if put_result.getFailedRecordCount() > 0
+          @logger.info "Put " + put_result.getFailedRecordCount().to_s + " records to datahub failed, total " + entries.size().to_s
+          sleep @retry_interval
+          entries = put_result.getFailedRecords()
+          @logger.info "write to datahub, failed: " + entries.size.to_s
+        else
+          @logger.info "Put data to datahub success, total " + entries.size().to_s
+        end
+      end
+
+    rescue DatahubPackage.exception.DatahubServiceException => e
+      @logger.error "Flush data exception: " + e.message #+ " " + e.backtrace.inspect.to_s
+      # shard的状态改变，需要重新加载shard
+      if e.getErrorCode() == "InvalidShardOperation"
+        @shards = get_active_shards(@topic.listShard())
+        @shard_count = @shards.size()
+
+        if @shard_count == 0
+          @logger.error "No active shard available, please check"
+        end
+      elsif e.getErrorCode() == nil
+        sleep @retry_interval
+      end
+      retry
+    rescue => e
+      @logger.error "Flush data exception: " + e.message + " " + e.backtrace.inspect.to_s
+
+      # 无限重试
+      if @retry_times < 0
+        @logger.warn "Now retry..."
+        # puts "Now retry..."
+        sleep @retry_interval
+        retry
+        # 重试次数用完
+      elsif @retry_times == 0
+        @logger.error "Retry not work, now exit"
+        Process.exit(1)
+        # 继续重试
+      elsif @retry_times > 0
+        @logger.warn "Now retry..."
+        # puts "Now retry..."
+        sleep @retry_interval
+        @retry_times -= 1
+        retry
+      end
+    end
+  end # def multi_receive
+
+end # class LogStash::Outputs::Datahub

data/logstash-output-datahub.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-datahub'
+  s.version         = "1.0.0"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "This aliyun-datahub output plugin."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Aliyun"]
+  s.email = "stream@service.aliyun.com"
+  s.homepage = "https://datahub.console.aliyun.com/datahub"
+  s.require_paths = ["lib"]
+  #s.platform    = 'java'
+  # Files
+  s.files = Dir['lib/**/*','lib/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/datahub.rb

@@ -0,0 +1,22 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/datahub"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::Datahub do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::Datahub.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-datahub
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Aliyun
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2016-09-20 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: stream@service.aliyun.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/outputs/datahub-test.rb
+- lib/logstash/outputs/datahub.rb
+- logstash-output-datahub.gemspec
+- spec/outputs/datahub.rb
+- vendor/jar-dependencies/runtime-jars/aliyun-sdk-datahub-2.2.1-SNAPSHOT.jar
+- vendor/jar-dependencies/runtime-jars/bouncycastle.provider-1.38-jdk15.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-io-2.4.jar
+- vendor/jar-dependencies/runtime-jars/commons-lang3-3.3.2.jar
+- vendor/jar-dependencies/runtime-jars/gson-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.4.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.4.4.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-asl-1.9.13.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.4.4.jar
+- vendor/jar-dependencies/runtime-jars/jackson-mapper-asl-1.9.13.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.12.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.12.jar
+homepage: https://datahub.console.aliyun.com/datahub
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: This aliyun-datahub output plugin.
+test_files:
+- spec/outputs/datahub.rb