logstash-input-pulsar 2.11.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: cc98f64cde7bf6bcf1b359fe61faf76cc92fc7c06f8e17e15d53e88a8ef97537
+  data.tar.gz: 3d755959c993856d6717ac7c1f47dc33a00827e6231efd25a51f15dd7ea41970
+SHA512:
+  metadata.gz: 7064fde6c6691c3093ca933422e6aa166b6e64e17e57c12c074ae3e4d6f163c27d79d8fddb51a45d3f10b320ce5a9762fdadb85648458a57b7e72929d6775df4
+  data.tar.gz: 605cf404da01e18f7ceb84e2e480863630369a474eccd50ca03b46ca0c877bd14b267e014d103a6f9680a7435d0618fb7d7f64addbeb9c808323b8dcf89287e4

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gem "logstash-devutils","=1.3.1"
+gemspec

data/README.md

@@ -0,0 +1,104 @@
+# Logstash Input Pulsar Plugin
+
+This is a Ruby plugin for [Logstash](https://github.com/elastic/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are free to use it however you want.
+
+This input will read events from a Pulsar topic.
+
+The version of this plugin x.y.z.m conforms to Pulsar x.y.z, while m is the patch version number. For broker compatibility, see the official Pulsar compatibility reference. If the compatibility wiki is not up-to-date, please contact Pulsar support/community to confirm compatibility.
+
+If you require features not yet available in this plugin (including client version upgrades), please file an issue with details about what you need.
+
+# Pulsar Input Configuration Options
+
+This plugin supports these configuration options.
+
+| Settings                      |                          Input type                          |     Default value | Required |
+| ----------------------------- | :----------------------------------------------------------: | ----------------: | -------: |
+| service_url                   |                            string                            |                 - |      Yes |
+| topics or topics_pattern      |                            array                             |       ["topic-1"] |       No |
+| subscription_name             |                            string                            |  "logstash-group" |       No |
+| client_id                     |                            string                            | "logstash-client" |       No |
+| subscription_type             | string, one of["Shared","Exclusive","Failover","Key_shared"] |          "Shared" |       No |
+| subscription_initial_position |             string, one of["Latest","Earliest"]              |        "Earliest" |       No |
+| codec                         |                            codec                             |           "plain" |       No |
+| consumer_threads              |                            number                            |                 1 |       No |
+| decorate_events               |                           boolean                            |             false |       No |
+| commit_async                  |                           boolean                            |             false |       No |
+| auth_plugin_class_name        |                            string                            |                 - |       No |
+| auth_params                   |                           password                           |                 - |       No |
+
+# Example
+
+pulsar without tls & token
+
+```
+input{
+  pulsar{
+    serviceUrl => "pulsar://127.0.0.1:6650"
+    codec => "json"
+    topics => [
+        "persistent://public/default/topic1",
+        "persistent://public/default/topic2"
+    ]
+    subscriptionName => "my_consumer"
+    subscriptionType => "Shared"
+    subscriptionInitialPosition => "Earliest"
+  }
+}
+```
+
+pulsar with token
+
+```
+input{
+  pulsar{
+    serviceUrl => "pulsar://127.0.0.1:6650"
+    codec => "plain"
+    topics => [
+        "persistent://public/default/topic1",
+        "persistent://public/default/topic2"
+    ]
+    subscriptionName => "my_subscription"
+    subscriptionType => "Shared"
+    subscriptionInitialPosition => "Earliest"
+    auth_plugin_class_name => "org.apache.pulsar.client.impl.auth.AuthenticationToken"
+    auth_params => "token:${token}"
+  }
+}
+```
+
+# Installation
+
+1. Get the latest zip file from release page.
+   https://github.com/NiuBlibing/logstash-input-pulsar/releases
+
+2. Install this plugin using logstash preoffline command.
+
+```
+bin/logstash-plugin install /{PATH_TO}/logstash-input-pulsar-x.y.z.m.gem
+```
+
+# Develop
+
+1. Develop environment
+
+- rvm
+- jruby (**_Do not_** use ruby, it may fail to compile)
+- gem
+- bundler
+- rake
+
+2. Install dependencies
+
+```sh
+bundle install
+rake install_jars
+```
+
+3. Build
+
+```sh
+gem build logstash-input-pulsar.gemspec
+```

data/lib/logstash/inputs/pulsar.rb

@@ -0,0 +1,139 @@
+require 'logstash/namespace'
+require 'logstash/inputs/base'
+require 'stud/interval'
+require 'java'
+require 'logstash-input-pulsar_jars'
+
+class LogStash::Inputs::Pulsar < LogStash::Inputs::Base
+  config_name "pulsar"
+
+  default :codec, 'plain'
+
+  config :service_url, :validate => :string, :required => true
+  config :auth_plugin_class_name, :validate => :string
+  config :auth_params, :validate => :password
+
+  config :topics, :validate => :array, :default => ["topic-1"]
+  config :topics_pattern, :validate => :string
+
+  config :subscription_name, :validate => :string, :default => "logstash-group"
+  config :client_id, :validate => :string, :default => "logstash-client"
+  config :subscription_type, :validate => :string, :default => "Shared"
+  config :subscription_initial_position, :validate => :string, :default => "Earliest"
+  config :consumer_threads, :validate => :number, :default => 1
+
+
+  config :decorate_events, :validate => :boolean, :default => false
+  config :commit_async, :validate => :boolean, :default => false
+
+
+  
+
+  public
+  def register
+    logger.info("register logstash-input-pulsar")
+    @runner_threads = []
+    @runner_pulsar_clients = Array.new
+  end # def register
+
+  public
+  def run(logstash_queue)
+    @runner_consumers = consumer_threads.times.map { |i| create_consumer("#{client_id}-#{i}") }
+    @runner_threads = @runner_consumers.map { |consumer| thread_runner(logstash_queue, consumer) }
+    @runner_threads.each { |t| t.join }
+  end # def run
+
+  # logstash 关闭回调
+  public
+  def stop
+    logger.info("Stop all pulsar consumer !")
+    @runner_consumers.each { |c| c.close }
+
+    logger.info("Stop all pulsar client  !!!")
+    @runner_pulsar_clients.each { |c| c.close }
+
+    @runner_threads.each { |t| t.exit }
+  end
+
+  private
+  def create_consumer(client_id)
+    begin
+      logger.info("client - ", :client => client_id)
+      clientBuilder = org.apache.pulsar.client.api.PulsarClient.builder()
+      clientBuilder.serviceUrl(@service_url)
+      if not @auth_plugin_class_name.nil? and not @auth_params.nil?
+        auth = org.apache.pulsar.client.api.AuthenticationFactory.create(@auth_plugin_class_name, @auth_params.value)
+        clientBuilder.authentication(auth)
+      end
+      client = clientBuilder.build
+      @runner_pulsar_clients.push(client)
+
+      logger.info("topic:",:topic => @topics.to_java('java.lang.String'))
+
+      consumerBuilder = client.newConsumer.clone
+      unless @topics_pattern.nil?
+        consumerBuilder.topicsPattern(@topics_pattern.to_java('java.lang.String'))
+      else
+        consumerBuilder.topic(@topics.to_java('java.lang.String')) 
+      end
+
+      subscriptionType = org.apache.pulsar.client.api.SubscriptionType
+      if @subscription_type == "Exclusive"
+        consumerBuilder.subscriptionType(subscriptionType::Exclusive)
+      elsif @subscription_type == "Failover"
+        consumerBuilder.subscriptionType(subscriptionType::Failover)
+      elsif @subscription_type == "Key_Shared"
+        consumerBuilder.subscriptionType(subscriptionType::Key_Shared)
+      else
+        consumerBuilder.subscriptionType(subscriptionType::Shared)
+      end
+
+      subscriptionInitialPositionType = org.apache.pulsar.client.api.SubscriptionInitialPosition
+      if @subscription_initial_position == "Latest"
+        consumerBuilder.subscriptionInitialPosition(subscriptionInitialPositionType::Latest)
+      else
+        consumerBuilder.subscriptionInitialPosition(subscriptionInitialPositionType::Earliest)
+      end
+
+      consumer = consumerBuilder.subscriptionName(@subscription_name)
+        .consumerName(client_id)
+        .subscribe();
+
+    rescue => e
+      logger.error("Unable to create pulsar consumer from given configuration",
+                   :pulsar_error_message => e,
+                   :cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+      throw e
+    end
+  end
+
+  private
+  def thread_runner(logstash_queue, consumer)
+    Thread.new do
+      begin
+        while !stop?
+          record = consumer.receive;
+          @codec.decode(record.getValue.to_s) do |event|
+            decorate(event)
+            if @decorate_events
+              event.set("[pulsar][topic]", record.getTopicName)
+              event.set("[pulsar][offset]", record.getMessageId)
+              event.set("[pulsar][key]", record.getKey)
+            end
+            logstash_queue << event
+            if commit_async
+              consumer.acknowledgeAsync(record)
+            else
+              consumer.acknowledge(record)
+            end
+            
+          end
+        end
+      rescue => e
+        logger.error("exit - ",:cause => e.respond_to?(:getCause) ? e.getCause() : nil)
+        consumer.close
+      end
+    end
+  end
+
+end

data/lib/logstash-input-pulsar_jars.rb

@@ -0,0 +1,5 @@
+# encoding: utf-8
+require 'logstash/environment'
+
+root_dir = File.expand_path(File.join(File.dirname(__FILE__), ".."))
+LogStash::Environment.load_runtime_jars! File.join(root_dir, "vendor")

data/logstash-input-pulsar.gemspec

@@ -0,0 +1,35 @@
+Gem::Specification.new do |s|
+    s.name            = 'logstash-input-pulsar'
+    s.version         = '2.11.0.3'
+    s.licenses        = ['Apache-2.0']
+    s.summary         = 'This input will read events from a pulsar topic.'
+    s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
+    s.authors         = ['ShiyiLiang', 'Tianyue Ren']
+    s.email           = 'rentianyue-jk@360shuke.com'
+    s.homepage        = "https://github.com/NiuBlibing/logstash-input-pulsar"
+    s.require_paths = ['lib']
+  
+    # Files
+    s.files = Dir["lib/**/*","spec/**/*","*.gemspec","*.md","CONTRIBUTORS","Gemfile","LICENSE","NOTICE.TXT", "vendor/jar-dependencies/**/*.jar", "vendor/jar-dependencies/**/*.rb", "VERSION", "docs/**/*"]
+  
+    # Tests
+    #s.test_files = s.files.grep(%r{^(test|spec|features)/})
+  
+    # Special flag to let us know this is actually a logstash plugin
+    s.metadata = { 'logstash_plugin' => 'true', 'group' => 'input'}
+  
+    s.requirements << "jar 'org.apache.pulsar:pulsar-client', '2.11.0'"
+    s.requirements << "jar 'org.slf4j:slf4j-log4j12', '1.7.32'"
+    s.requirements << "jar 'org.apache.logging.log4j:log4j-1.2-api', '2.18.0'"
+  
+    s.add_development_dependency 'jar-dependencies', '~> 0.4.1'
+  
+    # Gem dependencies
+    s.add_runtime_dependency "logstash-core-plugin-api", ">= 1.60", "<= 2.99"
+    s.add_runtime_dependency 'logstash-codec-json'
+    s.add_runtime_dependency 'logstash-codec-plain'
+    s.add_runtime_dependency 'stud', '>= 0.0.22', '< 0.1.0'
+  
+    s.add_development_dependency 'logstash-devutils'
+    s.add_development_dependency 'rspec-wait'
+  end

metadata

@@ -0,0 +1,217 @@
+--- !ruby/object:Gem::Specification
+name: logstash-input-pulsar
+version: !ruby/object:Gem::Version
+  version: 2.11.0.3
+platform: ruby
+authors:
+- ShiyiLiang
+- Tianyue Ren
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2023-03-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+  name: jar-dependencies
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.4.1
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+  name: logstash-core-plugin-api
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '1.60'
+    - - "<="
+      - !ruby/object:Gem::Version
+        version: '2.99'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-json
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+  name: stud
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.0.22
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: rspec-wait
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a Logstash plugin required to be installed on top of the
+  Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This
+  gem is not a stand-alone program
+email: rentianyue-jk@360shuke.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- lib/com/beust/jcommander/1.82/jcommander-1.82.jar
+- lib/com/github/luben/zstd-jni/1.3.7-3/zstd-jni-1.3.7-3.jar
+- lib/com/sun/activation/javax.activation/1.2.0/javax.activation-1.2.0.jar
+- lib/javax/validation/validation-api/1.1.0.Final/validation-api-1.1.0.Final.jar
+- lib/javax/ws/rs/javax.ws.rs-api/2.1/javax.ws.rs-api-2.1.jar
+- lib/log4j/log4j/1.2.17/log4j-1.2.17.jar
+- lib/logstash-input-pulsar_jars.rb
+- lib/logstash/inputs/pulsar.rb
+- lib/net/jcip/jcip-annotations/1.0/jcip-annotations-1.0.jar
+- lib/org/apache/logging/log4j/log4j-1.2-api/2.18.0/log4j-1.2-api-2.18.0.jar
+- lib/org/apache/logging/log4j/log4j-1.2-api/2.6.2/log4j-1.2-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-api/2.18.0/log4j-api-2.18.0.jar
+- lib/org/apache/logging/log4j/log4j-api/2.6.2/log4j-api-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-core/2.18.0/log4j-core-2.18.0.jar
+- lib/org/apache/logging/log4j/log4j-core/2.6.2/log4j-core-2.6.2.jar
+- lib/org/apache/logging/log4j/log4j-slf4j-impl/2.18.0/log4j-slf4j-impl-2.18.0.jar
+- lib/org/apache/pulsar/bouncy-castle-bc/2.11.0/bouncy-castle-bc-2.11.0-pkg.jar
+- lib/org/apache/pulsar/protobuf-shaded/2.1.0-incubating/protobuf-shaded-2.1.0-incubating.jar
+- lib/org/apache/pulsar/pulsar-client-admin-api/2.11.0/pulsar-client-admin-api-2.11.0.jar
+- lib/org/apache/pulsar/pulsar-client-api/2.11.0/pulsar-client-api-2.11.0.jar
+- lib/org/apache/pulsar/pulsar-client-api/2.3.0/pulsar-client-api-2.3.0.jar
+- lib/org/apache/pulsar/pulsar-client/2.11.0/pulsar-client-2.11.0.jar
+- lib/org/apache/pulsar/pulsar-client/2.3.0/pulsar-client-2.3.0.jar
+- lib/org/bouncycastle/bcpkix-jdk15on/1.60/bcpkix-jdk15on-1.60.jar
+- lib/org/bouncycastle/bcpkix-jdk15on/1.69/bcpkix-jdk15on-1.69.jar
+- lib/org/bouncycastle/bcprov-ext-jdk15on/1.69/bcprov-ext-jdk15on-1.69.jar
+- lib/org/bouncycastle/bcprov-jdk15on/1.60/bcprov-jdk15on-1.60.jar
+- lib/org/bouncycastle/bcprov-jdk15on/1.69/bcprov-jdk15on-1.69.jar
+- lib/org/bouncycastle/bcutil-jdk15on/1.69/bcutil-jdk15on-1.69.jar
+- lib/org/lz4/lz4-java/1.5.0/lz4-java-1.5.0.jar
+- lib/org/slf4j/slf4j-api/1.7.25/slf4j-api-1.7.25.jar
+- lib/org/slf4j/slf4j-api/1.7.32/slf4j-api-1.7.32.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.21/slf4j-log4j12-1.7.21.jar
+- lib/org/slf4j/slf4j-log4j12/1.7.32/slf4j-log4j12-1.7.32.jar
+- logstash-input-pulsar.gemspec
+- vendor/jar-dependencies/runtime-jars/bcpkix-jdk15on-1.69.jar
+- vendor/jar-dependencies/runtime-jars/bcprov-ext-jdk15on-1.69.jar
+- vendor/jar-dependencies/runtime-jars/bcprov-jdk15on-1.69.jar
+- vendor/jar-dependencies/runtime-jars/bcutil-jdk15on-1.69.jar
+- vendor/jar-dependencies/runtime-jars/bouncy-castle-bc-2.11.0-pkg.jar
+- vendor/jar-dependencies/runtime-jars/javax.activation-1.2.0.jar
+- vendor/jar-dependencies/runtime-jars/javax.ws.rs-api-2.1.jar
+- vendor/jar-dependencies/runtime-jars/jcip-annotations-1.0.jar
+- vendor/jar-dependencies/runtime-jars/jcommander-1.82.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2-api-2.18.0.jar
+- vendor/jar-dependencies/runtime-jars/log4j-1.2.17.jar
+- vendor/jar-dependencies/runtime-jars/log4j-api-2.18.0.jar
+- vendor/jar-dependencies/runtime-jars/log4j-core-2.18.0.jar
+- vendor/jar-dependencies/runtime-jars/log4j-slf4j-impl-2.18.0.jar
+- vendor/jar-dependencies/runtime-jars/pulsar-client-2.11.0.jar
+- vendor/jar-dependencies/runtime-jars/pulsar-client-admin-api-2.11.0.jar
+- vendor/jar-dependencies/runtime-jars/pulsar-client-api-2.11.0.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-api-1.7.32.jar
+- vendor/jar-dependencies/runtime-jars/slf4j-log4j12-1.7.32.jar
+- vendor/jar-dependencies/runtime-jars/validation-api-1.1.0.Final.jar
+homepage: https://github.com/NiuBlibing/logstash-input-pulsar
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  group: input
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements:
+- jar 'org.apache.pulsar:pulsar-client', '2.11.0'
+- jar 'org.slf4j:slf4j-log4j12', '1.7.32'
+- jar 'org.apache.logging.log4j:log4j-1.2-api', '2.18.0'
+rubygems_version: 3.3.25
+signing_key:
+specification_version: 4
+summary: This input will read events from a pulsar topic.
+test_files: []