logstash-output-datahub 1.0.0 → 1.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 41b554cbbdc8d2ea64dd05f8284660a499ce1346
-  data.tar.gz: df5877d7d4039a857c1fcaa2e5ba195ca7d0e7e7
+  metadata.gz: e846682c134462b56c1ee792a34143797b6b34ce
+  data.tar.gz: a36cd441f580ffd6763fc41cae69de4c7924f05b
 SHA512:
-  metadata.gz: c7fa8a628c8953db4af20fc7938879884484d092248c3e6bbbf476f785bec4509b305c55428f4b25b2603e572a10013143a35feb215248b17f4d8232a285fdf6
-  data.tar.gz: 59a9bb7aea19acea1f829b1891f9702141368764d2351cada3ecdb9e7471f1df82340ba5ba41f97b84ad31a36ff182479d753bff02bd41b4fcf1b6648824c99f
+  metadata.gz: eb309d8008b270ac7a8b485c57d48e2d85e3e548ec4d8d963fa4b318b7250b54f07a4cc26ce4ef67a18b80dda43ca4f589282765b2ab87de333093cc1189205c
+  data.tar.gz: b48d2a40c2677e0c262383765970fe63396bff623bd94449f95b1ebf04dbc6a0a5a902af4ccca2c01ebabafad1d5e9bd38efec8344ffdf0a9e73320b85f7f09b

data/README.md

@@ -71,8 +71,6 @@ output {
 		topic_name => ""
 		#shard_id => "0"
 		#shard_keys => ["thread_id"]
-		batch_size => 10
-		batch_timeout => 5
 		dirty_data_continue => true
 		dirty_data_file => "/Users/ph0ly/trash/dirty.data"
 		dirty_data_file_max_size => 1000
@@ -89,8 +87,6 @@ project_name(Required): datahub项目名称
 topic_name(Required): datahub topic名称
 retry_times(Optional): 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
 retry_interval(Optional): 下一次重试的间隔，单位为秒
-batch_size(Optional): 批量提交大小，指定数据积攒到@batch_size大小时触发一次提交，默认100
-batch_timeout(Optional): 批量提交超时，在数据量较少的情况下，数据超时后的超时提交，默认5秒
 shard_keys(Optional)：数组类型，数据落shard的字段名称，插件会根据这些字段的值计算hash将每条数据落某个shard, 注意shard_keys和shard_id都未指定，默认轮询落shard
 shard_id(Optional): 所有数据落指定的shard，注意shard_keys和shard_id都未指定，默认轮询落shard
 dirty_data_continue(Optional): 脏数据是否继续运行，默认为false，如果指定true，则遇到脏数据直接无视，继续处理数据。当开启该开关，必须指定@dirty_data_file文件

data/lib/logstash/outputs/datahub.rb

@@ -1,341 +1,357 @@
-#
-#Licensed to the Apache Software Foundation (ASF) under one
-#or more contributor license agreements.  See the NOTICE file
-#distributed with this work for additional information
-#regarding copyright ownership.  The ASF licenses this file
-#to you under the Apache License, Version 2.0 (the
-#"License"); you may not use this file except in compliance
-#with the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-#Unless required by applicable law or agreed to in writing,
-#software distributed under the License is distributed on an
-#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-#KIND, either express or implied.  See the License for the
-#specific language governing permissions and limitations
-#under the License.
-#
-require "logstash/outputs/base"
-require "logstash/namespace"
-require "logstash/environment"
-require "fileutils"
-require "thread"
-
-jar_path=File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
-LogStash::Environment.load_runtime_jars! File.join(jar_path, "vendor")
-
-# Datahub output plugin
-class LogStash::Outputs::Datahub < LogStash::Outputs::Base
-  declare_threadsafe!
-
-  config_name "datahub"
-
-  # datahub access id
-  config :access_id, :validate => :string, :required => true
-
-  # datahub access key
-  config :access_key, :validate => :string, :required => true
-
-  # datahub service endpoint
-  config :endpoint, :validate => :string, :required => true
-
-  # datahub project name
-  config :project_name, :validate => :string, :required => true
-
-  # datahub topic name
-  config :topic_name, :validate => :string, :required => true
-
-  # 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
-  config :retry_times, :validate => :number, :required => false, :default => -1
-
-  # 重试周期，下一次重试的间隔，单位为秒
-  config :retry_interval, :validate => :number, :required => false, :default => 5
-
-  # 按照指定字段的值计算hash，依据于该hash值落某个shard
-  config :shard_keys, :validate => :array, :required => false, :default => []
-
-  # 指定数据落指定的shard
-  config :shard_id, :validate => :string, :required => false, :default => ""
-
-  #   # 提交的列名，用户可以配置topic的列，采集部分列或者全部列
-  #   # 默认为空数组，表示按照topic的顺序及全字段提交
-  #   # 另外：列的配置不用保序，但是要求该字段在topic的schema中存在
-  #   config :column_names, :validate => :array, :required => false, :default => []
-
-  # 当出现脏数据时，是否继续写入
-  # 当开启该开关，必须指定@dirty_data_file文件
-  config :dirty_data_continue, :validate => :boolean, :required => false, :default => false
-
-  # 脏数据文件名称，当数据文件名称，在@dirty_data_continue开启的情况下，需要指定该值
-  # 特别注意：脏数据文件将被分割成两个部分.part1和.part2，part1作为更早的脏数据，part2作为更新的数据
-  config :dirty_data_file, :validate => :string, :required => false
-
-  # 脏数据文件的最大大小，该值保证脏数据文件最大大小不超过这个值，目前该值仅是一个参考值
-  config :dirty_data_file_max_size, :validate => :number, :required => false, :default => 50024000
-
-  # 数据传输压缩方式选择，目前支持deflate, lz4格式 
-  config :compress_method, :validate => :string, :required => false, :default => ""
-
-  # 该值内部使用，不提供配置
-  # 分发shard的游标
-  attr_accessor :shard_cursor
-
-  # Shard cursor lock
-  @@shard_lock = Mutex.new
-
-  # 写文件锁
-  @@file_lock = Mutex.new
-
-  DatahubPackage = com.aliyun.datahub
-
-  public
-  def register
-    begin
-      @account = DatahubPackage.auth.AliyunAccount::new(@access_id, @access_key)
-      @conf = DatahubPackage.DatahubConfiguration::new(@account, @endpoint)
-      if @compress_method == "deflate" || @compress_method == "lz4"
-        @compression_format = DatahubPackage.model.compress.CompressionFormat.fromValue(@compress_method)
-        @conf.setCompressionFormat(@compression_format)
-      end
-
-      @client = DatahubPackage.DatahubClient::new(@conf)
-      @project = DatahubPackage.wrapper.Project::Builder.build(@project_name, @client)
-      @topic = @project.getTopic(@topic_name)
-      @shard_cursor = 0
-
-      @shards = get_active_shards(@topic.listShard())
-      @shard_count = @shards.size()
-
-      result = @client.getTopic(@project_name, @topic_name)
-      @schema = result.getRecordSchema()
-      fields = @schema.getFields()
-      @columns_size = fields.size
-      @columns = []
-      for i in 0...@columns_size
-        @columns.push(fields[i].getName())
-      end
-
-      # 前置校验参数
-      check_params()
-
-      if @shard_count == 0
-        @logger.error "No active shard available, please check"
-        raise "No active shard available, please check"
-      end
-
-      @logger.info "Init datahub success!"
-    rescue => e
-      @logger.error "Init failed!"  + e.message + " " + e.backtrace.inspect.to_s
-      raise e
-    end
-  end # def register
-
-  def check_params()
-    # 如果shard_id配置了，则检查该shard是否ok
-    if !@shard_id.empty?
-      valid = false
-      for i in 0...@shards.size
-        shard_entry = @shards[i]
-        if shard_entry.getShardId() == @shard_id && shard_entry.getState() == DatahubPackage.model.ShardState::ACTIVE
-          valid = true
-        end
-      end
-      if (!valid)
-        @logger.error "Config shard_id not exists or state not active, check your config"
-        raise "Config shard_id not exists or state not active, check your config"
-      end
-    end
-
-    # 检查shard_keys字段是否合法
-    if @shard_keys.size > 0
-      for i in 0...@shard_keys.size
-        shard_key = @shard_keys[i]
-        if !@schema.containsField(shard_key)
-          @logger.error "Config shard_keys contains one or one more unknown field, check your config"
-          raise "Config shard_keys contains one or one more unknown field, check your config"
-        end
-      end
-    end
-
-    # 配置了脏数据继续，必须指定脏数据文件
-    if @dirty_data_continue
-      if @dirty_data_file.to_s.chomp.length == 0
-        raise "Dirty data file path can not be empty"
-      end
-    end
-
-  end
-
-  # 检查并设置数据到entry中
-  # 如果解析数据异常，则数据落脏数据文件
-  def check_and_set_data(entry, field_type, index, event_map, column_name)
-    data = event_map[column_name]
-    begin
-      if field_type == DatahubPackage.common.data.FieldType::STRING
-        entry.setString(index, data.to_s)
-      elsif field_type == DatahubPackage.common.data.FieldType::BIGINT
-        entry.setBigint(index, java.lang.Long.parseLong(data.to_s))
-      elsif field_type == DatahubPackage.common.data.FieldType::DOUBLE
-        entry.setDouble(index, java.lang.Double.parseDouble(data.to_s))
-      elsif field_type == DatahubPackage.common.data.FieldType::BOOLEAN
-        entry.setBoolean(index, java.lang.Boolean.parseBoolean(data.to_s))
-      elsif field_type == DatahubPackage.common.data.FieldType::TIMESTAMP
-        entry.setTimeStamp(index, java.lang.Long.parseLong(data.to_s))
-      else
-        raise "Unknown schema type of data"
-      end
-      return true
-    rescue => e
-      @logger.error "Parse data: " + column_name + "[" + data + "] failed, " + e.message
-      # 数据格式有异常，根据配置参数确定是否续跑
-      if !@dirty_data_continue
-        @logger.error "Dirty data found, exit process now."
-        puts "Dirty data found, exit process now."
-        Process.exit(1)
-        # 忽略的异常数据直接落文件
-      else
-        write_as_dirty_data(event_map)
-      end
-      return false
-    end
-  end
-
-  # 脏数据文件处理
-  def write_as_dirty_data(event_amp)
-    dirty_file_part1_name = @dirty_data_file + ".part1"
-    dirty_file_part2_name = @dirty_data_file + ".part2"
-
-    # 加锁写入
-    @@file_lock.synchronize {
-      dirty_file_part2 = File.open(dirty_file_part2_name, "a+")
-      dirty_file_part2.puts(event_amp.to_s)
-      dirty_file_part2.close
-      if File.size(dirty_file_part2_name) > @dirty_data_file_max_size / 2
-        # .part1, .part2分别存储数据
-        # 旧数据落part1，新的数据落part2
-        FileUtils.mv(dirty_file_part2_name, dirty_file_part1_name)
-      end
-    }
-  end
-
-  def get_active_shards(shards)
-    active_shards = []
-    for i in 0...shards.size
-      entry = shards.get(i)
-      if entry.getState() == DatahubPackage.model.ShardState::ACTIVE
-        active_shards.push(entry)
-      end
-    end
-    return active_shards
-  end
-
-  def get_next_shard_id()
-    if !@shard_id.empty?
-      return @shard_id
-      # 否则轮询写入shard
-    else
-      idx = 0
-      @@shard_lock.synchronize {
-        idx = @shard_cursor % @shard_count
-        @shard_cursor = idx + 1
-      }
-      shard_id = @shards[idx].getShardId()
-      return shard_id
-    end
-  end
-
-  def multi_receive(event_list)
-    begin
-      entries = []
-      shard_id = get_next_shard_id()
-
-      event_list.each do |event|
-        if event == LogStash::SHUTDOWN
-          return
-        end
-        event_map = event.to_hash
-
-        entry = DatahubPackage.model.RecordEntry::new(@schema)
-        #entry.putAttribute("srcId", event_map["host"].to_s)
-        #entry.putAttribute("ts", event_map["@timestamp"].to_s)
-        #entry.putAttribute("version", event_map["@version"].to_s)
-        #entry.putAttribute("srcType", "log")
-
-        for i in 0...@columns_size do
-          value = event_map[@columns[i]]
-          if value != nil
-            entry.set(i, value)
-          end
-        end
-
-        if @shard_keys.size > 0
-          hash_string = ""
-          for i in 0...@shard_keys.size
-            shard_key = @shard_keys[i]
-            if event_map[shard_key] != nil
-              hash_string += event_map[shard_key].to_s + ","
-            end
-          end
-          hashed_value = java.lang.String.new(hash_string).hashCode()
-          entry.setPartitionKey(hashed_value)
-        else
-          entry.setShardId(shard_id)
-        end
-        entries.push(entry)
-      end
-
-      # puts "total: " + entries.size.to_s
-
-      # 提交列表必须有数据
-      if entries.size > 0
-        put_result = @client.putRecords(@project_name, @topic_name, entries)
-        if put_result.getFailedRecordCount() > 0
-          @logger.info "Put " + put_result.getFailedRecordCount().to_s + " records to datahub failed, total " + entries.size().to_s
-          sleep @retry_interval
-          entries = put_result.getFailedRecords()
-          @logger.info "write to datahub, failed: " + entries.size.to_s
-        else
-          @logger.info "Put data to datahub success, total " + entries.size().to_s
-        end
-      end
-
-    rescue DatahubPackage.exception.DatahubServiceException => e
-      @logger.error "Flush data exception: " + e.message #+ " " + e.backtrace.inspect.to_s
-      # shard的状态改变，需要重新加载shard
-      if e.getErrorCode() == "InvalidShardOperation"
-        @shards = get_active_shards(@topic.listShard())
-        @shard_count = @shards.size()
-
-        if @shard_count == 0
-          @logger.error "No active shard available, please check"
-        end
-      elsif e.getErrorCode() == nil
-        sleep @retry_interval
-      end
-      retry
-    rescue => e
-      @logger.error "Flush data exception: " + e.message + " " + e.backtrace.inspect.to_s
-
-      # 无限重试
-      if @retry_times < 0
-        @logger.warn "Now retry..."
-        # puts "Now retry..."
-        sleep @retry_interval
-        retry
-        # 重试次数用完
-      elsif @retry_times == 0
-        @logger.error "Retry not work, now exit"
-        Process.exit(1)
-        # 继续重试
-      elsif @retry_times > 0
-        @logger.warn "Now retry..."
-        # puts "Now retry..."
-        sleep @retry_interval
-        @retry_times -= 1
-        retry
-      end
-    end
-  end # def multi_receive
-
-end # class LogStash::Outputs::Datahub
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require "logstash/outputs/base"
+require "logstash/namespace"
+require "logstash/environment"
+require "fileutils"
+require "thread"
+
+jar_path=File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
+LogStash::Environment.load_runtime_jars! File.join(jar_path, "vendor")
+
+# Datahub output plugin
+class LogStash::Outputs::Datahub < LogStash::Outputs::Base
+  declare_threadsafe!
+
+  config_name "datahub"
+
+  # datahub access id
+  config :access_id, :validate => :string, :required => true
+
+  # datahub access key
+  config :access_key, :validate => :string, :required => true
+
+  # datahub service endpoint
+  config :endpoint, :validate => :string, :required => true
+
+  # datahub project name
+  config :project_name, :validate => :string, :required => true
+
+  # datahub topic name
+  config :topic_name, :validate => :string, :required => true
+
+  # 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
+  config :retry_times, :validate => :number, :required => false, :default => -1
+
+  # 重试周期，下一次重试的间隔，单位为秒
+  config :retry_interval, :validate => :number, :required => false, :default => 5
+
+  # 按照指定字段的值计算hash，依据于该hash值落某个shard
+  config :shard_keys, :validate => :array, :required => false, :default => []
+
+  # 指定数据落指定的shard
+  config :shard_id, :validate => :string, :required => false, :default => ""
+
+  #   # 提交的列名，用户可以配置topic的列，采集部分列或者全部列
+  #   # 默认为空数组，表示按照topic的顺序及全字段提交
+  #   # 另外：列的配置不用保序，但是要求该字段在topic的schema中存在
+  #   config :column_names, :validate => :array, :required => false, :default => []
+
+  # 当出现脏数据时，是否继续写入
+  # 当开启该开关，必须指定@dirty_data_file文件
+  config :dirty_data_continue, :validate => :boolean, :required => false, :default => false
+
+  # 脏数据文件名称，当数据文件名称，在@dirty_data_continue开启的情况下，需要指定该值
+  # 特别注意：脏数据文件将被分割成两个部分.part1和.part2，part1作为更早的脏数据，part2作为更新的数据
+  config :dirty_data_file, :validate => :string, :required => false
+
+  # 脏数据文件的最大大小，该值保证脏数据文件最大大小不超过这个值，目前该值仅是一个参考值
+  config :dirty_data_file_max_size, :validate => :number, :required => false, :default => 50024000
+
+  # 数据传输压缩方式选择，目前支持deflate, lz4格式 
+  config :compress_method, :validate => :string, :required => false, :default => ""
+
+  # 该值内部使用，不提供配置
+  # 分发shard的游标
+  attr_accessor :shard_cursor
+
+  # Shard cursor lock
+  @@shard_lock = Mutex.new
+
+  # 写文件锁
+  @@file_lock = Mutex.new
+
+  DatahubPackage = com.aliyun.datahub
+
+  public
+  def register
+    begin
+      @account = DatahubPackage.auth.AliyunAccount::new(@access_id, @access_key)
+      @conf = DatahubPackage.DatahubConfiguration::new(@account, @endpoint)
+      if @compress_method == "deflate" || @compress_method == "lz4"
+        @compression_format = DatahubPackage.model.compress.CompressionFormat.fromValue(@compress_method)
+        @conf.setCompressionFormat(@compression_format)
+      end
+
+      @client = DatahubPackage.DatahubClient::new(@conf)
+      @project = DatahubPackage.wrapper.Project::Builder.build(@project_name, @client)
+      @topic = @project.getTopic(@topic_name)
+      @shard_cursor = 0
+
+      @shards = get_active_shards(@topic.listShard())
+      @shard_count = @shards.size()
+
+      result = @client.getTopic(@project_name, @topic_name)
+      @schema = result.getRecordSchema()
+      fields = @schema.getFields()
+      @columns_size = fields.size
+      @columnnames = []
+      for i in 0...@columns_size
+        @columnnames.push(fields[i].getName())
+      end
+      @columntypes = []
+      for i in 0...@columns_size
+        @columntypes.push(fields[i].getType())
+      end
+
+      # 前置校验参数
+      check_params()
+
+      if @shard_count == 0
+        @logger.error "No active shard available, please check"
+        raise "No active shard available, please check"
+      end
+
+      @logger.info "Init datahub success!"
+    rescue => e
+      @logger.error "Init failed!"  + e.message + " " + e.backtrace.inspect.to_s
+      raise e
+    end
+  end # def register
+
+  def check_params()
+    # 如果shard_id配置了，则检查该shard是否ok
+    if !@shard_id.empty?
+      valid = false
+      for i in 0...@shards.size
+        shard_entry = @shards[i]
+        if shard_entry.getShardId() == @shard_id && shard_entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+          valid = true
+        end
+      end
+      if (!valid)
+        @logger.error "Config shard_id not exists or state not active, check your config"
+        raise "Config shard_id not exists or state not active, check your config"
+      end
+    end
+
+    # 检查shard_keys字段是否合法
+    if @shard_keys.size > 0
+      for i in 0...@shard_keys.size
+        shard_key = @shard_keys[i]
+        if !@schema.containsField(shard_key)
+          @logger.error "Config shard_keys contains one or one more unknown field, check your config"
+          raise "Config shard_keys contains one or one more unknown field, check your config"
+        end
+      end
+    end
+
+    # 配置了脏数据继续，必须指定脏数据文件
+    if @dirty_data_continue
+      if @dirty_data_file.to_s.chomp.length == 0
+        raise "Dirty data file path can not be empty"
+      end
+    end
+
+  end
+
+  # 检查并设置数据到entry中
+  # 如果解析数据异常，则数据落脏数据文件
+  def check_and_set_data(entry, field_type, index, event_map, column_name)
+    data = event_map[column_name]
+    begin
+      if field_type == DatahubPackage.common.data.FieldType::STRING
+        entry.setString(index, data.to_s)
+      elsif field_type == DatahubPackage.common.data.FieldType::BIGINT
+        entry.setBigint(index, java.lang.Long.parseLong(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::DOUBLE
+        entry.setDouble(index, java.lang.Double.parseDouble(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::BOOLEAN
+        entry.setBoolean(index, java.lang.Boolean.parseBoolean(data.to_s))
+      elsif field_type == DatahubPackage.common.data.FieldType::TIMESTAMP
+        entry.setTimeStamp(index, java.lang.Long.parseLong(data.to_s))
+      else
+        raise "Unknown schema type of data"
+      end
+      return true
+    rescue => e
+      @logger.error "Parse data: " + column_name + "[" + data + "] failed, " + e.message
+      # 数据格式有异常，根据配置参数确定是否续跑
+      if !@dirty_data_continue
+        @logger.error "Dirty data found, exit process now."
+        puts "Dirty data found, exit process now."
+        Process.exit(1)
+        # 忽略的异常数据直接落文件
+      else
+        write_as_dirty_data(event_map)
+      end
+      return false
+    end
+  end
+
+  # 脏数据文件处理
+  def write_as_dirty_data(event_amp)
+    dirty_file_part1_name = @dirty_data_file + ".part1"
+    dirty_file_part2_name = @dirty_data_file + ".part2"
+
+    # 加锁写入
+    @@file_lock.synchronize {
+      dirty_file_part2 = File.open(dirty_file_part2_name, "a+")
+      dirty_file_part2.puts(event_amp.to_s)
+      dirty_file_part2.close
+      if File.size(dirty_file_part2_name) > @dirty_data_file_max_size / 2
+        # .part1, .part2分别存储数据
+        # 旧数据落part1，新的数据落part2
+        FileUtils.mv(dirty_file_part2_name, dirty_file_part1_name)
+      end
+    }
+  end
+
+  def get_active_shards(shards)
+    active_shards = []
+    for i in 0...shards.size
+      entry = shards.get(i)
+      if entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+        active_shards.push(entry)
+      end
+    end
+    return active_shards
+  end
+
+  def get_next_shard_id()
+    if !@shard_id.empty?
+      return @shard_id
+      # 否则轮询写入shard
+    else
+      idx = 0
+      @@shard_lock.synchronize {
+        idx = @shard_cursor % @shard_count
+        @shard_cursor = idx + 1
+      }
+      shard_id = @shards[idx].getShardId()
+      return shard_id
+    end
+  end
+
+  def multi_receive(event_list)
+    retry_count = 0
+    begin
+      entries = []
+      shard_id = get_next_shard_id()
+
+      event_list.each do |event|
+        if event == LogStash::SHUTDOWN
+          return
+        end
+        event_map = event.to_hash
+
+        entry = DatahubPackage.model.RecordEntry::new(@schema)
+        entry.putAttribute("srcId", event_map["host"].to_s)
+        entry.putAttribute("ts", event_map["@timestamp"].to_s)
+        entry.putAttribute("version", event_map["@version"].to_s)
+        entry.putAttribute("srcType", "log")
+
+        is_data_valid = false
+        for i in 0...@columns_size do
+          column_name = @columnnames[i]
+          column_type = @columntypes[i]
+          value = event_map[column_name]
+          if value != nil
+            is_data_valid = check_and_set_data(entry, column_type, i, event_map, column_name)
+            break if !is_data_valid
+          end
+        end
+
+        if is_data_valid
+          if @shard_keys.size > 0
+            hash_string = ""
+            for i in 0...@shard_keys.size
+              shard_key = @shard_keys[i]
+              if event_map[shard_key] != nil
+                hash_string += event_map[shard_key].to_s + ","
+              end
+            end
+            hashed_value = java.lang.String.new(hash_string).hashCode()
+            entry.setPartitionKey(hashed_value)
+          else
+            entry.setShardId(shard_id)
+          end
+          entries.push(entry)
+        end
+      end
+
+      # puts "total: " + entries.size.to_s
+
+      # 提交列表必须有数据
+      if entries.size > 0
+        put_result = @client.putRecords(@project_name, @topic_name, entries)
+        if put_result.getFailedRecordCount() > 0
+          @logger.info "Put " + put_result.getFailedRecordCount().to_s + " records to datahub failed, total " + entries.size().to_s
+          sleep @retry_interval
+          entries = put_result.getFailedRecords()
+          raise "Write to datahub failed: " + entries.size.to_s
+        else
+          @logger.info "Put data to datahub success, total " + entries.size().to_s
+        end
+      end
+
+    rescue DatahubPackage.exception.DatahubServiceException => e
+      @logger.error "Flush data exception: " + e.message #+ " " + e.backtrace.inspect.to_s
+      # shard的状态改变，需要重新加载shard
+      if e.getErrorCode() == "InvalidShardOperation"
+        @shards = get_active_shards(@topic.listShard())
+        @shard_count = @shards.size()
+
+        if @shard_count == 0
+          @logger.error "No active shard available, please check"
+        end
+      elsif e.getErrorCode() == nil
+        sleep @retry_interval
+      end
+      retry_count += 1
+      @logger.warn "Now retry: " + retry_count.to_s
+      retry
+    rescue => e
+      @logger.error "Flush data exception: " + e.message + " " + e.backtrace.inspect.to_s
+
+      # 无限重试
+      if @retry_times < 0
+        retry_count += 1
+        @logger.warn "Now retry: " + retry_count.to_s
+        # puts "Now retry..."
+        sleep @retry_interval
+        retry
+      elsif @retry_times == 0
+        @logger.error "Retry not work, now exit"
+        Process.exit(1)
+        # 继续重试
+      elsif @retry_times > 0
+        retry_count += 1
+        if retry_count > @retry_times
+          @logger.warn "Retry over: " + @retry_times.to_s
+          Process.exit(1)
+        end
+        @logger.warn "Now retry..."
+        sleep @retry_interval
+        retry
+      end
+    end
+  end # def multi_receive
+
+end # class LogStash::Outputs::Datahub

data/logstash-output-datahub.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-datahub'
-  s.version         = "1.0.0"
+  s.version         = "1.0.1"
   s.licenses = ["Apache License (2.0)"]
   s.summary = "This aliyun-datahub output plugin."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -19,7 +19,7 @@ Gem::Specification.new do |s|
 
   # Gem dependencies
   s.add_runtime_dependency 'stud'
-  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0"
   s.add_runtime_dependency "logstash-codec-plain"
   s.add_development_dependency "logstash-devutils"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-datahub
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.0.1
 platform: ruby
 authors:
 - Aliyun
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-09-20 00:00:00.000000000 Z
+date: 2017-06-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: stud
@@ -31,9 +31,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -41,9 +38,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: 2.0.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 3.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
@@ -126,7 +120,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: This aliyun-datahub output plugin.