fluent-plugin-nats-streaming 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: ece3ab243c73bbed8369ecba506784428edcd086
+  data.tar.gz: b087b636a06e1df13d0eaf6eca9eff402045ab68
+SHA512:
+  metadata.gz: 2c0925da021d693a45b7c0c73179dc9f810929e943d2db41d6e0c53309183ce18f20e69ce142050b9fe8d7ef0efbf5b2f2e27d94ea36b78f482b9bc63a58d85d
+  data.tar.gz: 667899467e6bd0564c7ad015d1d4c6c06f70b792a0cda5cd9785ab89d1e3f52aa6c13d0830e1f1f9b915375a360462f4fd18a384c542585f1c3aefd1cba3ce2a

data/.github/workflows/gempush.yml

@@ -0,0 +1,44 @@
+name: Ruby Gem
+
+on:
+  pull_request:
+    branches:
+      - master
+  push:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: Build + Publish
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@master
+    - name: Set up Ruby 2.6
+      uses: actions/setup-ruby@v1
+      with:
+        version: 2.6.x
+
+    - name: Publish to GPR
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:github: Bearer ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push --KEY github --host https://rubygems.pkg.github.com/${OWNER} *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.GPR_AUTH_TOKEN}}
+        OWNER: username
+
+    - name: Publish to RubyGems
+      run: |
+        mkdir -p $HOME/.gem
+        touch $HOME/.gem/credentials
+        chmod 0600 $HOME/.gem/credentials
+        printf -- "---\n:rubygems_api_key: ${GEM_HOST_API_KEY}\n" > $HOME/.gem/credentials
+        gem build *.gemspec
+        gem push *.gem
+      env:
+        GEM_HOST_API_KEY: ${{secrets.RUBYGEMS_AUTH_TOKEN}}

data/.gitignore

@@ -0,0 +1,19 @@
+*.gem
+*.rbc
+.bundle
+.config
+Gemfile.lock
+coverage
+InstalledFiles
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+
+# YARD artifacts
+.yardoc
+_yardoc
+doc/

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/README.md

@@ -0,0 +1,79 @@
+fluent-plugin-nats-streaming
+============================
+
+nats streaming plugin for [fluentd](https://github.com/fluent/fluentd) Event Collector
+
+# Getting Started
+example for nats streaming input:
+
+~~~~
+  <system>
+    workers 2
+  </system>
+
+  <source>
+    @type nats-streaming
+    server 127.0.0.1:4222
+    cluster_id test-cluster
+
+    # support multi child
+    client_id "in-#{Socket.gethostname}-#{worker_id}"
+    channel   nats.test
+    queue     test
+  </source>
+ 
+  <match nats.test>
+    @type stdout
+  </match>
+~~~~
+
+example for nats streaming output:
+
+~~~~
+  <system>
+    workers 2
+  </system>
+
+  <match nats.**>
+    @type nats-streaming
+    server 127.0.0.1:4222
+    client_id "out-#{Socket.gethostname}-#{worker_id}"
+    cluster_id test-cluster
+
+    <format>
+      @type json
+    </format>
+  </match>
+~~~~
+
+# Configuration
+* **server** (string) (optional): NATS streaming server host:port
+  * Default value: `localhost:4222`
+* **cluster_id** (string) (optional): cluster id 
+  * Default value: `fluentd`
+* **client_id** (string) (optional): client id 
+  * Default value: `fluentd`
+* **durable_name** (string) (optional): durable name
+  * Default value: `fluentd`
+* **queue** (string) (optional): queue name
+  * Default value: `fluentd`
+* **channel** (string) : channel name
+  * Default value: nil
+* **max_reconnect_attempts** (integer) : The max number of reconnect tries
+  * Default value: 10
+* **reconnect_time_wait** (integer) : The number of seconds to wait between reconnect tries
+  * Default value: 5
+* **timeout** (integer) : Ack timeout when publish
+  * Default value: 5
+
+
+
+
+
+
+
+
+
+
+
+

data/Rakefile

@@ -0,0 +1,13 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.test_files  = Dir['test/plugin/*.rb']
+  test.verbose = false
+  test.warning = false
+end
+
+task :default => :test

data/example/nats_in.conf

@@ -0,0 +1,19 @@
+<system>
+  log_level debug
+  workers 5
+</system>
+
+<source>
+  @type nats-streaming
+  server 127.0.0.1:4222
+  cluster_id test-cluster
+
+  # support multi child
+  client_id "in-#{Socket.gethostname}-#{worker_id}"
+  channel   nats.test
+  queue     test
+</source>
+
+<match nats.test>
+  @type stdout
+</match>

data/example/nats_out.conf

@@ -0,0 +1,37 @@
+<system>
+  log_level debug
+  workers 2
+</system>
+
+<source>
+  @type dummy
+  tag nats.test
+  auto_increment_key id
+  dummy {"hello":"world"}
+</source>
+
+<match nats.**>
+  @id nats
+  @type nats-streaming
+  server 127.0.0.1:4222
+  client_id "out-#{Socket.gethostname}-#{worker_id}"
+  cluster_id test-cluster
+
+  <format>
+    @type json
+  </format>
+
+  <buffer>
+    @type file
+    path /tmp/buffer/nats
+
+    flush_mode interval
+    flush_interval 1s
+    flush_thread_count 5
+    flush_at_shutdown true
+
+    retry_type periodic
+    retry_forever true
+    retry_wait 1m
+  </buffer>
+</match>

data/fluent-plugin-nats-streaming.gemspec

@@ -0,0 +1,26 @@
+# $:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name        = "fluent-plugin-nats-streaming"
+  gem.version     = "0.0.1"
+  gem.authors     = ["hc chien"]
+  gem.email       = ["hc.chien@pentium.network"]
+  gem.homepage    = "https://github.com/hc-chien/fluent-plugin-nats-streaming.git"
+  gem.summary     = %q{nats streaming plugin for fluentd, an event collector}
+  gem.description = %q{nats streaming plugin for fluentd, an event collector}
+  gem.license     = "Apache-2.0"
+
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.require_paths = ["lib"]
+  gem.required_ruby_version = ">= 2.4.0"
+
+  gem.add_dependency "fluentd", ">= 0.14.20", "< 2"
+#  nats-0.11.
+#  gem.add_dependency "nats", '~> 0.11', ">= 0.11.0"
+  gem.add_dependency "nats-streaming", '~> 0.2', ">= 0.2.2"
+
+  gem.add_development_dependency "rake", '~> 0.9', ">= 0.9.2"
+  gem.add_development_dependency "test-unit", '~> 0.3', "> 3.1"
+end

data/lib/fluent/plugin/in_nats-streaming.rb

@@ -0,0 +1,99 @@
+require "fluent/plugin/input"
+require 'stan/client'
+
+module Fluent::Plugin
+  class NatsStreamingInput < Input
+
+    Fluent::Plugin.register_input('nats-streaming', self)
+
+    helpers :thread
+
+    config_param :server, :string, :default => 'localhost:4222',
+                 :desc => "NATS streaming server host:port"
+    config_param :cluster_id, :string, :default => 'fluentd',
+                 :desc => "cluster id"
+    config_param :client_id, :string, :default => 'fluentd',
+                 :desc => "client id"
+    config_param :durable_name, :string, :default => 'fluentd',
+                 :desc => "durable name"
+    config_param :queue, :string, :default => 'fluentd',
+                 :desc => "queue name"
+    config_param :channel, :string, :default => nil,
+                 :desc => "channel name"
+
+    config_param :max_reconnect_attempts, :integer, :default => 10,
+                 :desc => "The max number of reconnect tries"
+    config_param :reconnect_time_wait, :integer, :default => 5,
+                 :desc => "The number of seconds to wait between reconnect tries"
+
+    def multi_workers_ready?
+      true
+    end
+
+    def initialize
+      super
+      @sc = nil
+    end
+
+    def configure(conf)
+      super
+
+      @sc_config = {
+        servers: ["nats://#{server}"],
+        reconnect_time_wait: @reconnect_time_wait,
+        max_reconnect_attempts: @max_reconnect_attempts
+      }
+
+      @sub_opts = {
+        queue: @queue,
+        durable_name: @durable_name,
+        start_at: :first,
+        deliver_all_available: true,
+        ack_wait: 10,  # seconds
+        connect_timeout: 2 # seconds
+      }
+    end
+
+    def start
+      super
+      thread_create(:nats_streaming_input_main, &method(:run))
+    end
+
+    def run
+      @sc = STAN::Client.new
+
+      log.info "connect nats server nats://#{server} #{cluster_id} #{client_id}"
+      @sc.connect(@cluster_id, @client_id.gsub(/\./, '_'), nats: @sc_config)
+      log.info "connected"
+
+      log.info "subscribe #{channel} #{queue} #{durable_name}"
+      @sc.subscribe(@channel, @sub_opts) do |msg|
+        tag = @channel
+        begin
+          message = JSON.parse(msg.data)
+        rescue JSON::ParserError => e
+          log.error "Failed parsing JSON #{e.inspect}.  Passing as a normal string"
+          message = msg.data
+        end
+        time = Fluent::Engine.now
+        router.emit(tag, time, message || {})
+      end
+
+      while thread_current_running?
+        log.trace "test connection"
+        @sc.nats.flush(@reconnect_time_wait)
+        sleep(5)
+      end
+    end
+
+    def close
+      super
+      @sc.close if @sc
+    end
+
+    def terminate
+      super
+      @sc = nil
+    end
+  end
+end

data/lib/fluent/plugin/out_nats-streaming.rb

@@ -0,0 +1,151 @@
+require "fluent/plugin/output"
+require 'stan/client'
+
+module Fluent::Plugin
+  class NatsStreamingOutput < Output
+
+    Fluent::Plugin.register_output('nats-streaming', self)
+
+    helpers :formatter, :thread, :inject 
+
+    DEFAULT_FORMAT_TYPE = 'json'
+
+    config_param :server, :string, :default => 'localhost:4222',
+                 :desc => "NATS streaming server host:port"
+    config_param :cluster_id, :string, :default => 'fluentd',
+                 :desc => "cluster id"
+    config_param :client_id, :string, :default => 'fluentd',
+                 :desc => "client id"
+    config_param :durable_name, :string, :default => nil,
+                 :desc => "durable name"
+
+    config_param :max_reconnect_attempts, :integer, :default => 10,
+                 :desc => "The max number of reconnect tries"
+    config_param :reconnect_time_wait, :integer, :default => 5,
+                 :desc => "The number of seconds to wait between reconnect tries"
+    config_param :connect_timeout, :integer, :default => 2,
+                 :desc => "Connect timeout in seconds"
+    config_param :timeout, :integer, :default => 5,
+                 :desc => "Ack timeout"
+
+    config_section :buffer do
+      config_set_default :@type, 'memory'
+      config_set_default :flush_mode, :interval
+      config_set_default :flush_interval, 1
+      config_set_default :chunk_keys, ['tag']
+      config_set_default :flush_at_shutdown, true
+      config_set_default :chunk_limit_size, 10 * 1024
+    end
+
+    config_section :format do
+      config_set_default :@type, DEFAULT_FORMAT_TYPE
+      config_set_default :add_newline, false
+    end
+
+    def multi_workers_ready?
+      true
+    end
+
+    def formatted_to_msgpack_binary?
+      true
+    end
+
+    def initialize
+      super
+      @sc = nil
+    end
+
+    def configure(conf)
+      super
+
+      @sc_config = {
+        servers: ["nats://#{server}"],
+        reconnect_time_wait: @reconnect_time_wait,
+        max_reconnect_attempts: @max_reconnect_attempts,
+        connect_timeout: @connect_timeout
+      }
+
+      formatter_conf = conf.elements('format').first
+      unless formatter_conf
+        raise Fluent::ConfigError, "<format> section is required."
+      end
+      unless formatter_conf["@type"]
+        raise Fluent::ConfigError, "format/@type is required."
+      end
+      @formatter_proc = setup_formatter(formatter_conf)
+    end
+
+    def start
+      super
+      thread_create(:nats_streaming_output_main, &method(:run))
+    end
+
+    def run
+      @sc = STAN::Client.new
+
+      log.info "connect nats server nats://#{server} #{cluster_id} #{client_id}"
+      @sc.connect(@cluster_id, @client_id.gsub(/\./, '_'), nats: @sc_config)
+      log.info "connected"
+
+      while thread_current_running?
+        log.trace "test connection"
+        @sc.nats.flush(@reconnect_time_wait)
+        sleep(5)
+      end
+    end
+
+    def setup_formatter(conf)
+      type = conf['@type']
+      case type
+      when 'json'
+        begin
+          require 'oj'
+          Oj.default_options = Fluent::DEFAULT_OJ_OPTIONS
+          Proc.new { |tag, time, record| Oj.dump(record) }
+        rescue LoadError
+          require 'yajl'
+          Proc.new { |tag, time, record| Yajl::Encoder.encode(record) }
+        end
+      when 'ltsv'
+        require 'ltsv'
+        Proc.new { |tag, time, record| LTSV.dump(record) }
+      else
+        @formatter = formatter_create(usage: 'kafka-plugin', conf: conf)
+        @formatter.method(:format)
+      end
+    end
+
+    def process(tag, es)
+      es = inject_values_to_event_stream(tag, es)
+      es.each do |time,record|
+        @sc.publish(tag, format(tag, time, record))
+      end
+    end
+
+    def write(chunk)
+      return if chunk.empty?
+      tag = chunk.metadata.tag
+
+      messages = 0
+      chunk.each { |time, record|
+        record_buf = @formatter_proc.call(tag, time, record)
+        log.trace "Send record: #{record_buf}"
+        @sc.publish(tag, record_buf, {timeout: @timeout} )
+        messages += 1
+      }
+      if messages > 0
+          log.debug { "#{messages} messages send." }
+      end
+    end
+
+    def close
+      super
+      @sc.close if @sc
+    end
+
+    def terminate
+      super
+      @sc = nil
+    end
+  end
+end

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-nats-streaming
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- hc chien
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2019-11-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.20
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.14.20
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: nats-streaming
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.2.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.9'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.9.2
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.3'
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '3.1'
+description: nats streaming plugin for fluentd, an event collector
+email:
+- hc.chien@pentium.network
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".github/workflows/gempush.yml"
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- example/nats_in.conf
+- example/nats_out.conf
+- fluent-plugin-nats-streaming.gemspec
+- lib/fluent/plugin/in_nats-streaming.rb
+- lib/fluent/plugin/out_nats-streaming.rb
+homepage: https://github.com/hc-chien/fluent-plugin-nats-streaming.git
+licenses:
+- Apache-2.0
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.4.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.7
+signing_key: 
+specification_version: 4
+summary: nats streaming plugin for fluentd, an event collector
+test_files: []