fluent-plugin-nsq-local 0.0.7.dev.4

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: b113f08ba211549199e711cec9471db5fbefc6b9
+  data.tar.gz: 26a802fb77649360ffd9824128967e212983f6ab
+SHA512:
+  metadata.gz: 157d4adbc16da363cf502183e17ca7f6e0624c59788e23e2d23fc892905e5ee5f79f8c090558465919b595164c1aef946df05999e3d6bcc01ecd9d632ef7b4a6
+  data.tar.gz: 7a6591bfb251766fb67fbbc5d36646e867df704ee9c46111d1bb58494d2c70da91b4d3e8f5e567aad7e6bc523600ebcdd134781a2b40193ef5adacf09d18e1bd

data/fluent-plugin-nsq.gemspec

@@ -0,0 +1,25 @@
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name          = "fluent-plugin-nsq-local"
+  s.version       = `cat VERSION`
+  s.authors       = ["lxfontes", "dterror"]
+  s.email         = ["lucas@uken.com", "diogo@uken.com"]
+  s.description   = %q{NSQ output plugin for Fluentd}
+  s.summary       = %q{output plugin for fluentd}
+  s.homepage      = "https://github.com/uken/fluent-plugin-nsq"
+  s.license       = 'MIT'
+
+  git_files = `git ls-files`.split($/)
+
+  s.files         = git_files.grep(%r{^(lib|fluent|bin)})
+  s.executables   = git_files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  s.test_files    = git_files.grep(%r{^(test|spec|features)/})
+  s.require_paths = ["lib"]
+
+  s.add_runtime_dependency 'fluentd', ['> 0.14', '< 2']
+  s.add_runtime_dependency 'nsq-ruby', '~> 2.3'
+  s.add_development_dependency 'rake', '~> 10'
+  s.add_development_dependency 'json', '~> 2'
+  s.add_development_dependency("test-unit", ["~> 3.2"])
+end

data/lib/fluent/plugin/in_nsq.rb

@@ -0,0 +1,95 @@
+# coding: utf-8
+
+require 'fluent/plugin/input'
+require 'nsq'
+require 'yajl'
+
+module Fluent::Plugin
+  class NSQInput < Input
+    Fluent::Plugin.register_input('nsq', self)
+
+    config_param :topic, :string, default: nil
+    config_param :channel, :string, default: 'fluent_nsq_input'
+    config_param :in_flight, :integer, default: 100
+    config_param :nsqlookupd, :array, default: nil
+    config_param :tag, :string, default: '_key'
+    config_param :time_key, :string, default: nil
+    config_param :tag_source, default: :key do |val|
+      case val.downcase
+      when 'key'
+        :key
+      when 'topic'
+        :topic
+      when 'static'
+        :static
+      else
+        fail Fluent::ConfigError, 'tag_source should be either "key", "static" or "topic"'
+      end
+    end
+
+    def configure(conf)
+      super
+
+      fail Fluent::ConfigError, 'Missing nsqlookupd' unless @nsqlookupd
+      fail Fluent::ConfigError, 'Missing topic' unless @topic
+      fail Fluent::ConfigError, 'Missing channel' unless @channel
+      fail Fluent::ConfigError, 'in_flight needs to be bigger than 0' unless @in_flight > 0
+    end
+
+    def start
+      super
+      @consumer = Nsq::Consumer.new(
+        nsqlookupd: @nsqlookupd,
+        topic: @topic,
+        channel: @channel,
+        max_in_flight: @in_flight
+      )
+      @running = true
+      @thread = Thread.new(&method(:consume))
+    end
+
+    def shutdown
+      super
+      @running = false
+      @consumer.terminate
+    end
+
+    private
+    def consume
+      while @running
+        consume_one
+      end
+    end
+
+    def consume_one
+      msg = @consumer.pop
+      record = Yajl.load(msg.body.force_encoding('UTF-8'))
+      record_tag = tag_for_record(record)
+      record_time = time_for_record(record, msg)
+      router.emit(record_tag, record_time, record)
+      msg.finish
+    rescue => e
+      log.warn("nsq: #{e}")
+      msg.requeue if msg
+    end
+
+    def tag_for_record(record)
+      case @tag_source
+      when :static
+        @tag
+      when :key
+        record[@tag]
+      when :topic
+        @topic
+      end
+    end
+
+    def time_for_record(record, msg)
+      if @time_key
+        record[@time_key]
+      else
+        Fluent::EventTime.new(msg.timestamp.to_i, msg.timestamp.nsec)
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_nsq.rb

@@ -0,0 +1,79 @@
+# coding: utf-8
+
+require 'fluent/plugin/output'
+require 'nsq'
+require 'yajl'
+
+module Fluent::Plugin
+  class NSQOutput < Output
+    Fluent::Plugin.register_output('nsq', self)
+
+    config_param :topic, :string, default: nil
+    config_param :nsqlookupd, :array, default: nil
+    config_param :nsqd, :array, default: nil
+    config_param :enable_tls, :bool, default: false
+    config_param :key, :string, default: nil
+    config_param :certificate, :string, default: nil
+    config_param :ca_certificate, :string, default: nil
+
+    config_section :buffer do
+      config_set_default :chunk_keys, ['tag']
+    end
+
+    def configure(conf)
+      super
+
+      fail Fluent::ConfigError, 'Missing nsqlookupd or nsqd' unless @nsqlookupd || @nsqd
+      fail Fluent::ConfigError, 'Missing topic' unless @topic
+      if @enable_tls
+        fail Fluent::ConfigError, 'Missing tls config params' unless @key && @certificate && @ca_certificate
+      end
+    end
+
+    def start
+      super
+      producer_config = {topic: @topic}
+
+      if  @nsqlookupd
+        producer_config[:nsqlookupd] = @nsqlookupd
+      elsif @nsqd
+        producer_config[:nsqd] = @nsqd
+      end
+
+      if @enable_tls
+        producer_config[:tls_v1] = true
+        producer_config[:tls_options] = {
+              key: @key,
+              certificate: @certificate,
+              ca_certificate: @ca_certificate,
+              verify_mode: OpenSSL::SSL::VERIFY_PEER
+          }
+      end
+      
+      @producer = Nsq::Producer.new(producer_config)
+    end
+
+    def shutdown
+      @producer.terminate
+      super
+    end
+
+    def write(chunk)
+      return if chunk.empty?
+
+      tag = chunk.metadata.tag
+      chunk.each do |time, record|
+        tagged_record = record.merge(
+          :_key => tag,
+          :_ts => time.to_f,
+          :'@timestamp' => Time.at(time).iso8601(3) # kibana/elasticsearch friendly
+        )
+        begin
+          @producer.write(Yajl.dump(tagged_record))
+        rescue => e
+          log.warn("nsq: #{e}")
+        end
+      end
+    end
+  end
+end

data/test/plugin/test_in_nsq.rb

@@ -0,0 +1,66 @@
+require 'test/unit'
+
+require 'fluent/test'
+require 'fluent/test/driver/input'
+require 'fluent/plugin/in_nsq'
+
+require 'date'
+
+require 'helper'
+
+$:.push File.expand_path("../lib", __FILE__)
+$:.push File.dirname(__FILE__)
+
+class TestNSQInput < Test::Unit::TestCase
+  TCONFIG = %[
+    nsqlookupd localhost:4161
+    topic logs_in
+    time_key _ts
+  ]
+  def setup
+    #Nsq.logger = Logger.new(STDOUT)
+    Fluent::Test.setup
+  end
+
+  def test_configure
+    d = create_driver
+    assert_not_nil d.instance.topic
+  end
+
+  def create_driver(conf=TCONFIG)
+    Fluent::Test::Driver::Input.new(Fluent::Plugin::NSQInput).configure(conf)
+  end
+
+  def create_producer
+    Nsq::Producer.new(
+      nsqlookupd: ['127.0.0.1:4161'],
+      topic: 'logs_in'
+    )
+  end
+
+  def sample_record
+    {_ts: Time.now, _key: 'somekey', age:26, request_id: '42', parent_id: 'parent', sub: {field: {pos: 15}}}
+  end
+
+  def test_wrong_config
+    assert_raise Fluent::ConfigError do
+      create_driver('')
+    end
+  end
+
+  def test_sample_record_loop
+    d = create_driver
+    d.run do
+      prod = create_producer
+      sleep(1)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      sleep(1)
+      prod.terminate
+    end
+    puts("emitz")
+    puts(d.events)
+  end
+end

data/test/plugin/test_out_nsq.rb

@@ -0,0 +1,51 @@
+require 'test/unit'
+
+require 'fluent/test'
+require 'fluent/test/driver/output'
+require 'fluent/plugin/out_nsq'
+
+require 'date'
+
+require 'helper'
+
+$:.push File.expand_path("../lib", __FILE__)
+$:.push File.dirname(__FILE__)
+
+class TestNSQOutput < Test::Unit::TestCase
+  TCONFIG = %[
+    nsqlookupd localhost:4161
+    topic logs_out
+  ]
+  def setup
+    #Nsq.logger = Logger.new(STDOUT)
+    Fluent::Test.setup
+  end
+
+  def test_configure
+    d = create_driver
+    assert_not_nil d.instance.topic
+  end
+
+  def create_driver(conf=TCONFIG)
+    Fluent::Test::Driver::Output.new(Fluent::Plugin::NSQOutput).configure(conf)
+  end
+
+  def sample_record
+    {'age' => 26, 'request_id' => '42', 'parent_id' => 'parent', 'sub' => {'field'=>{'pos'=>15}}}
+  end
+
+  def test_wrong_config
+    assert_raise Fluent::ConfigError do
+      create_driver('')
+    end
+  end
+
+  def test_sample_record_loop
+    d = create_driver
+    d.run(default_tag: 'test') do
+      100.times.each do |t|
+        d.feed(sample_record)
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,131 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-nsq-local
+version: !ruby/object:Gem::Version
+  version: 0.0.7.dev.4
+platform: ruby
+authors:
+- lxfontes
+- dterror
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-06-01 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">"
+      - !ruby/object:Gem::Version
+        version: '0.14'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: nsq-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10'
+- !ruby/object:Gem::Dependency
+  name: json
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+- !ruby/object:Gem::Dependency
+  name: test-unit
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.2'
+description: NSQ output plugin for Fluentd
+email:
+- lucas@uken.com
+- diogo@uken.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- fluent-plugin-nsq.gemspec
+- lib/fluent/plugin/in_nsq.rb
+- lib/fluent/plugin/out_nsq.rb
+- test/helper.rb
+- test/plugin/test_in_nsq.rb
+- test/plugin/test_out_nsq.rb
+homepage: https://github.com/uken/fluent-plugin-nsq
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">"
+    - !ruby/object:Gem::Version
+      version: 1.3.1
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.14.4
+signing_key: 
+specification_version: 4
+summary: output plugin for fluentd
+test_files:
+- test/helper.rb
+- test/plugin/test_in_nsq.rb
+- test/plugin/test_out_nsq.rb