logstash-input-datahub 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 2e8c3f6b66cc1841ca687384bf582539f0e1006d
+  data.tar.gz: ce8b7a82469a07a16e11fb323c801e57c8716be3
+SHA512:
+  metadata.gz: 8ec7e67535eb392a3193c46ed74ba8cc2893b70ab81512dc9d1bda71e74b41ed9abb4b62df75b8fa905c8faec7e0d529db37cf56fcd5a4255c78dfb968985d0a
+  data.tar.gz: e81b90beb17a3f3587755bf86581c4f7561b45756d33de62fd3099bbd7ba07bc9d4b17ffc0e69cc9199085ce3cc0f1c89d027e75fa1b1b2582f02f658aa5caf2

data/CHANGELOG.md

@@ -0,0 +1,2 @@
+0.0.1
+First version.

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012–2015 Elasticsearch <http://www.elastic.co>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,86 @@
+# Aliyun DataHub Plugin for LogStash
+
+## Getting Started
+---
+
+### 介绍
+
+- 该插件是基于logstash开发的输入插件，它主要完成消费DataHub(阿里云产品)服务上的数据。
+
+### 安装
++ 环境要求linux, jdk1.7+, logstash（可选，如果没安装也没关系）
++ 从流计算官网下载tar包，使用以下命令安装
+
+如果之前没安装过logstash，请使用以下步骤安装
+
+```
+$ tar -xzvf logstash-with-datahub-2.3.0.tar.gz
+$ cd logstash-with-datahub-2.3.0
+```
+
+如果之前安装过logstash，拿到logstash-input-datahub-1.0.0.gem，再使用以下命令安装
+
+```
+$ ${LOGSTASH_HOME}/bin/logstash-plugin install --local logstash-input-datahub-1.0.0.gem
+```
+
+### 样例
+logstash的配置如下：
+
+```
+input {
+    datahub {
+        access_id => "Your accessId"
+        access_key => "Your accessKey"
+        endpoint => "http://dh-cn-hangzhou.aliyuncs.com"
+        project_name => "test_project"
+        topic_name => "test_logstash"
+        interval=> 5
+        #cursor => {
+        #    "0"=>"20000000000000000000000003110091"
+        #    "2"=>"20000000000000000000000003110091"
+        #    "1"=>"20000000000000000000000003110091"
+        #    "4"=>"20000000000000000000000003110091"
+        #    "3"=>"2000000000000000000000000311000"
+        #}
+        shard_ids => []
+        pos_file => "/home/admin/logstash/logstash-2.4.0/pos_file"
+    }
+}
+
+output {
+    file {
+        path => "/home/admin/logstash/logstash-2.4.0/output"
+    }
+}
+```
+
+### 参数介绍
+```
+access_id(Required): 阿里云access id
+access_key(Required): 阿里云access key
+endpoint(Required): 阿里云datahub的服务地址
+project_name(Required): datahub项目名称
+topic_name(Required): datahub topic名称
+retry_times(Optional): 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
+retry_interval(Optional): 下一次重试的间隔，单位为秒
+shard_ids(Optional)：数组类型，需要消费的shard列表，空列表默认全部消费
+cursor(Optional)：消费起点，默认为空，表示从头开始消费
+pos_file(Required)：checkpoint记录文件，必须配置，优先使用checkpoint恢复消费offset
+```
+
+## 相关参考
+---
+
+- [LogStash主页](https://www.elastic.co/products/logstash)
+- [LogStash插件开发](https://www.elastic.co/guide/en/logstash/current/_how_to_write_a_logstash_input_plugin.html#_coding_input_plugins)
+
+## Authors && Contributors
+---
+
+- [Dong Xiao](https://github.com/dongxiao1198)
+
+## License
+---
+
+licensed under the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0.html)

data/lib/logstash/inputs/datahub-test.rb

@@ -0,0 +1,10 @@
+require "test/unit"
+require "datahub"
+
+class DatahubTest < Test::Unit::TestCase
+
+    def test_register()
+        
+    end
+    
+end

data/lib/logstash/inputs/datahub.rb

@@ -0,0 +1,348 @@
+#
+#Licensed to the Apache Software Foundation (ASF) under one
+#or more contributor license agreements.  See the NOTICE file
+#distributed with this work for additional information
+#regarding copyright ownership.  The ASF licenses this file
+#to you under the Apache License, Version 2.0 (the
+#"License"); you may not use this file except in compliance
+#with the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+#Unless required by applicable law or agreed to in writing,
+#software distributed under the License is distributed on an
+#"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#KIND, either express or implied.  See the License for the
+#specific language governing permissions and limitations
+#under the License.
+#
+require "logstash/inputs/base"
+require "logstash/namespace"
+require "logstash/environment"
+require "thread"
+require "stud/interval"
+
+jar_path=File.expand_path(File.join(File.dirname(__FILE__), "../../.."))
+LogStash::Environment.load_runtime_jars! File.join(jar_path, "vendor")
+
+# Datahub output plugin
+class LogStash::Inputs::Datahub < LogStash::Inputs::Base
+  config_name "datahub"
+  default :codec, "plain"
+  # datahub access id
+  config :access_id, :validate => :string, :required => true
+
+  # datahub access key
+  config :access_key, :validate => :string, :required => true
+
+  # datahub service endpoint
+  config :endpoint, :validate => :string, :required => true
+
+  # datahub project name
+  config :project_name, :validate => :string, :required => true
+
+  # datahub topic name
+  config :topic_name, :validate => :string, :required => true
+
+  # 重试次数，-1为无限重试、0为不重试、>0表示需要有限次数
+  config :retry_times, :validate => :number, :required => false, :default => -1
+
+  # 重试周期，下一次重试的间隔，单位为秒
+  config :retry_interval, :validate => :number, :required => false, :default => 3
+
+  # 指定shard
+  config :shard_ids, :validate => :array, :required => false, :default => []
+
+  # 指定start cursor
+  config :cursor, :validate => :hash, :required => false, :default => {}
+
+  # seek from cursor time
+  config :cursor_time, :validate => :hash, :required => false, :default => {}
+
+  # 消费batch
+  config :batch_limit, :validate => :number, :default => 100
+
+  # 消费间隔
+  config :interval, :validate => :number, :default => 2
+
+  # 消费间隔
+  config :pos_file, :validate => :string, :required => true
+
+  # 使用多线程
+  config :multi_thread, :validate => :boolean, :required => false, :default => true
+
+  # 写文件锁
+  @@mutex = Mutex.new
+
+  @@consumer_threads = []
+
+  DatahubPackage = com.aliyun.datahub
+
+  public
+  def register
+    begin
+      @stop = false
+      @account = DatahubPackage.auth.AliyunAccount::new(@access_id, @access_key)
+      @conf = DatahubPackage.DatahubConfiguration::new(@account, @endpoint)
+
+      @client = DatahubPackage.DatahubClient::new(@conf)
+      @project = DatahubPackage.wrapper.Project::Builder.build(@project_name, @client)
+      @topic = @project.getTopic(@topic_name)
+
+      @shards = get_active_shards(@topic.listShard())
+      @shard_count = @shards.size()
+
+      result = @client.getTopic(@project_name, @topic_name)
+      @schema = result.getRecordSchema()
+      @fields = @schema.getFields()
+
+      # 前置校验参数
+      check_params()
+      # 读取checkpoint
+      read_checkpoint()
+
+      if @shard_count == 0
+        @logger.error "No active shard available, please check"
+        raise "No active shard available, please check"
+      end
+
+      @logger.info "Init datahub success!"
+    rescue => e
+      @logger.error "Init failed!"  + e.message + " " + e.backtrace.inspect.to_s
+      raise e
+    end
+  end # def register
+
+  def check_params()
+    @logger.info "Config shardIds:" + @shard_ids.to_s
+    if @shard_ids.size != 0
+      valid = true
+      for shard in 0...@shard_ids.size
+        @logger.info "Checking shard:" + @shard_ids[shard]
+        shard_exist_active = false
+        for i in 0...@shards.size
+          shard_entry = @shards[i]
+          if shard_entry.getShardId() == @shard_ids[shard] && shard_entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+            shard_exist_active = true
+            break
+          end
+        end
+        if !shard_exist_active
+          valid = false
+        end
+      end
+      if (!valid)
+        @logger.error "Config shard_id not exists or state not active, check your config"
+        raise "Config shard_id not exists or state not active, check your config"
+      end
+    else
+      valid = false
+      for i in 0...@shards.size
+        shard_entry = @shards[i]
+        @logger.info "Checking shard:" + shard_entry.getShardId()
+        if shard_entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+          @shard_ids.push(shard_entry.getShardId())
+          valid = true
+        end
+      end
+      if (!valid)
+        @logger.error "There is no active shard."
+        raise "There is no active shard."
+      end
+    end
+    @logger.info "Reading from shards:" + @shard_ids.to_s
+    begin
+      checkpoint_file = File.open(@pos_file, "a+")
+      checkpoint_file.close
+    rescue
+      @logger.error "Config pos file is invalid, pos_file must point to a file."
+      raise "Config pos file is invalid, pos_file must point to a file."
+    end
+  end
+
+  def update_checkpoint()
+    @logger.info "flush checkpoint:" + @cursor.to_s
+    @@mutex.synchronize {
+      checkpoint_file = File.open(@pos_file, "w")
+      @cursor.each { |key,value|
+        checkpoint_file.write(key+":"+value+"\n")
+      }
+      checkpoint_file.close
+    }
+  end
+
+  def read_checkpoint()
+    begin
+      @logger.info "read checkpoint:" + @pos_file
+      @@mutex.synchronize {
+        File.foreach(@pos_file) do |line|
+          checkpoint = line.chomp
+          cursor_param = checkpoint.split(":")
+          if cursor_param.size != 2
+            raise "Invalid checkpoint:" + checkpoint
+          end
+          @cursor[cursor_param[0]] = cursor_param[1]
+        end
+        @logger.info "recover checkpoint:" + @cursor.to_s
+      }
+    rescue => e
+      @logger.error e.backtrace.inspect.to_s
+      @logger.error "No checkpoint found or invalid, will start normally."
+      raise e
+    end
+  end
+
+  def update_cursor(shard_id, cursor)
+    @@mutex.synchronize {
+      @cursor[shard_id] = cursor
+    }
+  end
+
+  def get_cursor(shard_id, force = false)
+    @@mutex.synchronize {
+      if (force || !@cursor.has_key?(shard_id) || @cursor[shard_id]==nil || @cursor[shard_id]=="")
+        @logger.info "Shard:" + shard_id + " has no checkpoint, will seek to begin."
+	if (@cursor_time.has_key?(shard_id))
+		if (@cursor_time[shard_id] == '-1')
+                    cursorRs = @client.getCursor(@project_name, @topic_name, shard_id, DatahubPackage.model.GetCursorRequest::CursorType::LATEST)
+	        else
+            	    cursorRs = @client.getCursor(@project_name, @topic_name, shard_id, @cursor_time[shard_id].to_i * 1000)
+		end
+	else
+            cursorRs = @client.getCursor(@project_name, @topic_name, shard_id, DatahubPackage.model.GetCursorRequest::CursorType::OLDEST)
+	end	
+        @cursor[shard_id] = cursorRs.getCursor()
+        @logger.info "Start reading shard:" + shard_id + " with cursor:" + @cursor[shard_id]
+      end
+      return @cursor[shard_id]
+    }
+  end
+
+  def read_record(shard_id, queue)
+    cursor = get_cursor(shard_id)
+    begin
+      @logger.info "Get record at shard: " + shard_id + " cursor:" + cursor
+      recordRs = @client.getRecords(@project_name, @topic_name, shard_id, cursor, @batch_limit, @schema)
+      recordEntries = recordRs.getRecords()
+      recordEntries.each do |record|
+        data = Hash.new
+        @fields.each do |field|
+          case field.getType()
+            when DatahubPackage.common.data.FieldType::BIGINT
+              data[field.getName()] = record.getBigint(field.getName())
+            when DatahubPackage.common.data.FieldType::DOUBLE
+              data[field.getName()] = record.getDouble(field.getName())
+            when DatahubPackage.common.data.FieldType::BOOLEAN
+              data[field.getName()] = record.getBoolean(field.getName())
+            when DatahubPackage.common.data.FieldType::TIMESTAMP
+              data[field.getName()] = record.getTimeStamp(field.getName())
+            when DatahubPackage.common.data.FieldType::STRING
+              data[field.getName()] = record.getString(field.getName())
+            else
+              @logger.error "Unknow type " + field.getType().toString()
+              raise "Unknow type " + field.getType().toString()
+          end
+        end
+        data["timestamp"] = record.getSystemTime()
+        event = LogStash::Event.new("message"=>data)
+        decorate(event)
+        queue << event
+      end
+      cursor = recordRs.getNextCursor()
+      update_cursor(shard_id, cursor)
+      @logger.info "Shard:" + shard_id + " Next cursor:" + cursor + " GetRecords:" + (recordRs.getRecordCount()).to_s
+      if (recordRs.getRecordCount() == 0)
+        @logger.info "Read to end, waiting for data:" + cursor
+        Stud.stoppable_sleep(@interval) { stop? }
+      else
+        update_checkpoint
+      end
+    rescue DatahubPackage.exception.InvalidCursorException => e
+      @logger.error "Shard:" + shard_id + " Invalid cursor:" + cursor + ", seek to begin."
+      get_cursor(shard_id, true)
+    rescue DatahubPackage.exception.InvalidOperationException => e
+      @logger.error "Shard:" + shard_id + " Invalid operation, cursor:" + cursor + ", shard is sealed, consumer will exit."
+      raise e
+    rescue DatahubPackage.exception.DatahubClientException => e
+      @logger.error "Read failed:" + e.getMessage() + ", will retry later."
+      Stud.stoppable_sleep(@retry_interval) { stop? }
+    end
+  end
+
+  def shard_consumer(shard_id, queue)
+    @logger.info "Consumer thread started:" + shard_id
+    while !check_stop do
+      begin
+        read_record(shard_id, queue)
+      rescue DatahubPackage.exception.InvalidOperationException => e
+        @logger.error "Shard:" + shard_id + " Invalid operation caused by sealed shard, consumer will exit."
+        break
+      rescue => e
+        @logger.error "Read records failed" + e.backtrace.inspect.to_s
+        @logger.error "Will retry after " + @retry_interval.to_s + " seconds."
+        Stud.stoppable_sleep(@retry_interval) { stop? }
+      end
+    end # loop
+    @logger.info "Consumer thread exit, shard:" + shard_id
+  end
+
+  def check_stop()
+    @@mutex.synchronize {
+      return @stop
+    }
+  end
+
+  def run(queue)
+    if @multi_thread
+      for i in 0...@shard_ids.size
+        @@consumer_threads << Thread.new(@shard_ids[i],queue) {|shard_id,queue|
+          shard_consumer(shard_id, queue)
+        }
+      end
+      while !check_stop do
+        Stud.stoppable_sleep(15) { stop? }
+        @logger.debug "Main thread heartbeat."
+      end # loop
+    else
+      while !check_stop do
+        for i in 0...@shard_ids.size
+          begin
+            read_record(@shard_ids[i], queue)
+          rescue DatahubPackage.exception.InvalidOperationException => e
+            @logger.error "Shard:" + @shard_ids[i] + " Invalid operation caused by sealed shard, remove this shard."
+            @shard_ids.delete_at(i)
+            break
+          rescue => e
+            @logger.error "Read records failed" + e.backtrace.inspect.to_s
+            @logger.error "Will retry after " + @retry_interval.to_s + " seconds."
+            Stud.stoppable_sleep(@retry_interval) { stop? }
+          end
+        end
+      end # loop
+    end
+    @logger.info "Main thread exit."
+  end # def run
+
+  def get_active_shards(shards)
+    active_shards = []
+    for i in 0...shards.size
+      entry = shards.get(i)
+      if entry.getState() == DatahubPackage.model.ShardState::ACTIVE
+        active_shards.push(entry)
+      end
+    end
+    return active_shards
+  end
+
+  def stop
+    @@mutex.synchronize {
+      @stop = true
+    }
+    @logger.info "Plugin stopping, waiting for consumer thread stop."
+    @@consumer_threads.each { |thread|
+      thread.join
+    }
+    @logger.info "Plugin stopped."
+  end
+
+end # class LogStash::Inputs::Datahub

data/logstash-input-datahub.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-input-datahub'
+  s.version         = "1.0.0"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "This aliyun-datahub input plugin."
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Aliyun"]
+  s.email = "stream@service.aliyun.com"
+  s.homepage = "https://datahub.console.aliyun.com/datahub"
+  s.require_paths = ["lib"]
+  #s.platform    = 'java'
+  # Files
+  s.files = Dir['lib/**/*','lib/*','spec/**/*','vendor/**/*','*.gemspec','*.md','CONTRIBUTORS','Gemfile','LICENSE']
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'stud'
+  s.add_runtime_dependency "logstash-core", ">= 2.0.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+end

data/spec/input/datahub.rb

@@ -0,0 +1,11 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/input/datahub"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Inputs::Datahub do
+  it_behaves_like "test input plugin" do
+    let(:config) { { "interval" => 100 } }
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-datahub
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Aliyun
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2017-02-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: stud
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not
+  a stand-alone program
+email: stream@service.aliyun.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- CONTRIBUTORS
+- Gemfile
+- LICENSE
+- README.md
+- lib/logstash/inputs/datahub-test.rb
+- lib/logstash/inputs/datahub.rb
+- logstash-input-datahub.gemspec
+- spec/input/datahub.rb
+- vendor/jar-dependencies/runtime-jars/aliyun-sdk-datahub-2.2.1-SNAPSHOT.jar
+- vendor/jar-dependencies/runtime-jars/bouncycastle.provider-1.38-jdk15.jar
+- vendor/jar-dependencies/runtime-jars/commons-codec-1.9.jar
+- vendor/jar-dependencies/runtime-jars/commons-io-2.4.jar
+- vendor/jar-dependencies/runtime-jars/commons-lang3-3.3.2.jar
+- vendor/jar-dependencies/runtime-jars/gson-2.6.2.jar
+- vendor/jar-dependencies/runtime-jars/jackson-annotations-2.4.0.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-2.4.4.jar
+- vendor/jar-dependencies/runtime-jars/jackson-core-asl-1.9.13.jar
+- vendor/jar-dependencies/runtime-jars/jackson-databind-2.4.4.jar
+- vendor/jar-dependencies/runtime-jars/jackson-mapper-asl-1.9.13.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/lz4-1.3.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.12.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.12.jar
+homepage: https://datahub.console.aliyun.com/datahub
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: input
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5.1
+signing_key: 
+specification_version: 4
+summary: This aliyun-datahub input plugin.
+test_files:
+- spec/input/datahub.rb