fluent-plugin-nsq 0.0.3 → 0.0.4

data/lib/fluent/plugin/in_nsq.rb

@@ -0,0 +1,95 @@
+# coding: utf-8
+
+module Fluent
+  class NSQInput < Input
+    Plugin.register_input('nsq', self)
+
+    config_param :topic, :string, default: nil
+    config_param :channel, :string, default: 'fluent_nsq_input'
+    config_param :nsqlookupd, :string, default: nil
+    config_param :tag, :string, default: '_key'
+    config_param :time_key, :string, default: nil
+    config_param :tag_source, default: :key do |val|
+      case val.downcase
+      when 'key'
+        :key
+      when 'topic'
+        :topic
+      when 'static'
+        :static
+      else
+        fail ConfigError, 'tag_source should be either "key", "static" or "topic"'
+      end
+    end
+
+    def initialize
+      super
+      require 'cool.io'
+      require 'nsq'
+    end
+
+    def configure(conf)
+      super
+
+      fail ConfigError, 'Missing nsqlookupd' unless @nsqlookupd
+      fail ConfigError, 'Missing topic' unless @topic
+      fail ConfigError, 'Missing channel' unless @channel
+    end
+
+    def start
+      super
+      lookupds = @nsqlookupd.split(',')
+      @consumer = Nsq::Consumer.new(
+        nsqlookupd: lookupds,
+        topic: @topic,
+        channel: @channel
+      )
+      @running = true
+      @thread = Thread.new(&method(:consume))
+    end
+
+    def shutdown
+      super
+      @running = false
+      @consumer.terminate
+    end
+
+    private
+    def consume
+      while @running
+        consume_one
+      end
+    end
+
+    def consume_one
+      msg = @consumer.pop
+      record = JSON.parse(msg.body)
+      record_tag = tag_for_record(record)
+      record_time = time_for_record(record, msg)
+      Engine.emit(record_tag, record_time, record)
+      msg.finish
+    rescue => e
+      log.warn("nsq: #{e}")
+      msg.requeue if msg
+    end
+
+    def tag_for_record(record)
+      case @tag_source
+      when :static
+        @tag
+      when :key
+        record[@tag]
+      when :topic
+        @topic
+      end
+    end
+
+    def time_for_record(record, msg)
+      if @time_key
+        record[@time_key]
+      else
+        msg.timestamp
+      end
+    end
+  end
+end

data/lib/fluent/plugin/out_nsq.rb

@@ -1,5 +1,4 @@
 # coding: utf-8
-require 'nsq'
 
 module Fluent
   class NSQOutput < BufferedOutput
@@ -10,6 +9,7 @@ module Fluent
 
     def initialize
       super
+      require 'nsq'
     end
 
     def configure(conf)

data/test/plugin/test_in_nsq.rb

@@ -0,0 +1,65 @@
+require 'test/unit'
+
+require 'fluent/test'
+require 'fluent/plugin/in_nsq'
+
+require 'date'
+
+require 'helper'
+
+$:.push File.expand_path("../lib", __FILE__)
+$:.push File.dirname(__FILE__)
+
+class TestNSQInput < Test::Unit::TestCase
+  TCONFIG = %[
+    nsqlookupd localhost:4161
+    topic logs_in
+    time_key _ts
+  ]
+  def setup
+    #Nsq.logger = Logger.new(STDOUT)
+    Fluent::Test.setup
+  end
+
+  def test_configure
+    d = create_driver
+    assert_not_nil d.instance.topic
+  end
+
+  def create_driver(conf=TCONFIG)
+    Fluent::Test::InputTestDriver.new(Fluent::NSQInput).configure(conf)
+  end
+
+  def create_producer
+    Nsq::Producer.new(
+      nsqlookupd: ['127.0.0.1:4161'],
+      topic: 'logs_in'
+    )
+  end
+
+  def sample_record
+    {_ts: Time.now, _key: 'somekey', age:26, request_id: '42', parent_id: 'parent', sub: {field: {pos: 15}}}
+  end
+
+  def test_wrong_config
+    assert_raise Fluent::ConfigError do
+      create_driver('')
+    end
+  end
+
+  def test_sample_record_loop
+    d = create_driver
+    d.run do
+      prod = create_producer
+      sleep(1)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      prod.write(sample_record.to_json)
+      sleep(1)
+      prod.terminate
+    end
+    puts("emitz")
+    puts(d.emits)
+  end
+end

data/test/plugin/test_out_nsq.rb

@@ -13,7 +13,7 @@ $:.push File.dirname(__FILE__)
 class TestNSQOutput < Test::Unit::TestCase
   TCONFIG = %[
     nsqlookupd localhost:4161
-    topic logs
+    topic logs_out
   ]
   def setup
     #Nsq.logger = Logger.new(STDOUT)
@@ -35,7 +35,7 @@ class TestNSQOutput < Test::Unit::TestCase
 
   def test_wrong_config
     assert_raise Fluent::ConfigError do
-      d = create_driver('test','')
+      create_driver('test','')
     end
   end
 

metadata

@@ -1,7 +1,8 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-nsq
 version: !ruby/object:Gem::Version
-  version: 0.0.3
+  version: 0.0.4
+  prerelease: 
 platform: ruby
 authors:
 - lxfontes
@@ -9,11 +10,12 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-11-05 00:00:00.000000000 Z
+date: 2014-11-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -21,6 +23,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -28,6 +31,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nsq-ruby
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -35,6 +39,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -42,6 +47,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -49,6 +55,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -62,33 +69,37 @@ extensions: []
 extra_rdoc_files: []
 files:
 - fluent-plugin-nsq.gemspec
+- lib/fluent/plugin/in_nsq.rb
 - lib/fluent/plugin/out_nsq.rb
 - test/helper.rb
+- test/plugin/test_in_nsq.rb
 - test/plugin/test_out_nsq.rb
 homepage: https://github.com/uken/fluent-plugin-nsq
 licenses:
 - MIT
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.3.0
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: output plugin for fluentd
 test_files:
 - test/helper.rb
+- test/plugin/test_in_nsq.rb
 - test/plugin/test_out_nsq.rb

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    ODc2NjIxMzUyYWVmZDY1MDYyMTk0ODQwMDczNjE3ZDM0Y2QxNzA1Yg==
-  data.tar.gz: !binary |-
-    MTFmYjVkODU5M2VkYjg2MzFmYjY5YjM0MWUzZTc0ZTdlYWQ0NzNhOA==
-SHA512:
-  metadata.gz: !binary |-
-    YTkwMzgyMjc4NTlhM2ZkYjE5N2JlMzFjNDQxZWFjZDNlYjcyNDZhM2FmM2E4
-    ZTJhMGIwMDc5ZGQyYTE5MjY1YWNhMmMxNWRkMWJiYjIyNDllMzFiNmY5MTA2
-    NjQ2MWI1YzBjYzkwYzAyNzAzMDdmNDkyYzNjYzY1NjIxMWQ2ZTI=
-  data.tar.gz: !binary |-
-    OGFiMzg0N2ZiYzJmOGRkM2Y3ZTIwZWEwYTE5NmYxM2FkZDRlNzdjZTEzM2Zj
-    NjFiM2YyM2M3ZDRhZDI5NzA5NGIxNjI5NTU4NzU5NGFmNmM2YTQ4MDRlMjky
-    ZWRkMjcyMzg0ZWU5YTIzOGUxODMyNDZiN2VmZTcxMzE0ZmRkZjE=