fluent-plugin-http-pull 0.1.0 → 0.2.0.pre

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 3dba2cd5721def7eb2a06c6c555ac86ed7e71af7
-  data.tar.gz: 7c236824e3f147f3834cb48808b29d0a21a8a197
+  metadata.gz: 198fe9ad32356188705d13ad28b5587e956a9dae
+  data.tar.gz: 1bd9d9eb636748fbd24e5ea58b687138990d3021
 SHA512:
-  metadata.gz: 44b6b63cc58a1fd1fc7811cbba3aa92597db70827b2acbbe327b778217f8760696c2e6d7a79837eaada4da0e2a3e41267c5a393600aae85bc5052799efa1c4b3
-  data.tar.gz: 606d3ba7b39fd1c8966215599dca7fa0e63b30267c7f782b902b8a2fa81a9a9cd22ed38adea50657061b63e528402fb9c70571a99ee6b7ce476874c901574740
+  metadata.gz: d28286e0838a18b75954d315beea25d9da6831fff29b28d2ad7d5471505e7e758ceff63e5dc85dbd7cf2069325bb0692582c7a7d0ec7e07561a49d43d22faf37
+  data.tar.gz: 55fe7c398310a9fbc4596a57753a43e3970a8c5091507396bcda160e3765a4e7c223a5f333e2b5d4e94e7b09df08505e18f0fd628bc7717892c207b01a47151d

data/README.md

@@ -1,5 +1,9 @@
 # fluent-plugin-http-pull
 
+[![Build Status](https://travis-ci.org/HatsuneMiku3939/fluent-plugin-http-pull.svg?branch=master)](https://travis-ci.org/HatsuneMiku3939/fluent-plugin-http-pull)
+[![Gem Version](https://badge.fury.io/rb/fluent-plugin-http-pull.svg)](https://badge.fury.io/rb/fluent-plugin-http-pull)
+
+
 [Fluentd](http://fluentd.org/) input plugin to pull log from rest api.
 
 Many of modern server application offer status reporting API via http (even 'fluentd' too). This plugin will help to gathering status log from these status api.
@@ -61,21 +65,19 @@ $ bundle
 <source>
   @type http_pull
 
-  tag test
+  tag fluentd.status
   url http://localhost:24220/api/plugins.json
   interval 1
-
-  status_only false
 </source>
 
-<match test>
+<match fluentd.status>
   @type stdout
 </match>
 
-# 2017-05-17 21:41:47.872951000 +0900 test: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
-# 2017-05-17 21:41:48.955316000 +0900 test: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
-# 2017-05-17 21:41:50.033628000 +0900 test: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
-# 2017-05-17 21:41:51.107372000 +0900 test: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
+# 2017-05-17 21:41:47.872951000 +0900 fluentd.status: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
+# 2017-05-17 21:41:48.955316000 +0900 fluentd.status: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
+# 2017-05-17 21:41:50.033628000 +0900 fluentd.status: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
+# 2017-05-17 21:41:51.107372000 +0900 fluentd.status: {"url":"http://localhost:24220/api/plugins.json","status":200,"message":{"plugins":[{"plugin_id":"object:1e7e3d...
 ```
 
 
@@ -84,44 +86,60 @@ $ bundle
 <source>
   @type http_pull
 
-  tag test
+  tag es.cluster.health
   url http://localhost:9200/_cluster/health
   interval 1
-
-  status_only false
 </source>
 
-<match test>
+<match es.cluster.health>
   @type stdout
 </match>
 
-# 2017-05-17 12:49:09.886298008 +0000 test: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
-# 2017-05-17 12:49:10.669431296 +0000 test: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
-# 2017-05-17 12:49:11.668789668 +0000 test: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
-# 2017-05-17 12:49:12.668789849 +0000 test: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
+# 2017-05-17 12:49:09.886298008 +0000 es.cluster.health: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
+# 2017-05-17 12:49:10.669431296 +0000 es.cluster.health: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
+# 2017-05-17 12:49:11.668789668 +0000 es.cluster.health: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
+# 2017-05-17 12:49:12.668789849 +0000 es.cluster.health: {"url":"http://localhost:9200/_cluster/health","status":200,"message":{"cluster_name":"elasticsearch","status":"green",...
 ```
 
 ## Configuration
 
-* See also: Fluent::Plugin::Input
-
-## Fluent::Plugin::HttpPullInput
-
 ### tag (string) (required)
 
 The tag of the event.
 
 ### url (string) (required)
 
-The uri of monitoring target
+The url of remote server.
 
 ### interval (integer) (required)
 
-The second interval time between periodic request
+The second interval time between periodic request.
 
-### status_only (bool) (optional)
+### status_only (bool) (optional, default: false)
 
-status_only
+If atatus_only is true, body is not parsed.
+
+## In case of remote error
+
+### Can receive response from remote
+
+```
+{
+  "url": url of remote
+  "status": status code of response
+  "error": "RestClient::NotFound: 404 Not Found" or something similar
+}
+```
+
+### All the other case
+
+```
+{
+  "url": url of remote
+  "status": 0
+  "error": "Errno::ECONNREFUSED: Connection refused - connect(2) for "localhost" port 12345" or something similar
+}
+```
 
 ## Copyright
 

data/fluent-plugin-http-pull.gemspec

@@ -3,7 +3,7 @@ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 
 Gem::Specification.new do |spec|
   spec.name    = "fluent-plugin-http-pull"
-  spec.version = "0.1.0"
+  spec.version = "0.2.0.pre"
   spec.authors = ["filepang"]
   spec.email   = ["filepang@gmail.com"]
 
@@ -22,6 +22,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "test-unit", "~> 3.0"
+  spec.add_development_dependency "test-unit-rr", "~> 1.0", "~> 1.0.2"
 
   spec.add_runtime_dependency "fluentd", [">= 0.14.10", "< 2"]
   spec.add_runtime_dependency "rest-client", [">= 2.0.0", "< 3"]

data/lib/fluent/plugin/in_http_pull.rb

@@ -47,20 +47,25 @@ module Fluent
         while !@shutdown
           ts = Time.now.to_f
 
-          res = RestClient.get(@url)
-          if @status_only
-            router.emit(@tag, Engine.now, {
-              "url" => @url,
-              "status" => res.code
-            })
-          else
-            router.emit(@tag, Engine.now, {
-              "url" => @url,
-              "status" => res.code,
-              "message" => JSON.parse(res.body)
-            })
+          log = { "url" => @url }
+
+          begin
+            res = RestClient.get(@url)
+            log["status"] = res.code
+            log["body"] = res.body
+          rescue RestClient::ExceptionWithResponse => err
+            log["status"] = err.code
+            log["error"] = err.message
+          rescue Exception => err
+            log["status"] = 0
+            log["error"] = err.message
           end
 
+          log["message"] = JSON.parse(log["body"]) if !@status_only && log["body"] != nil
+          log.delete("body")
+
+          router.emit(@tag, Engine.now, log)
+
           te = Time.now.to_f
           delay = @interval - (te - ts)
           delay = delay > 0 ? delay : 0

data/test/helper.rb

@@ -4,5 +4,7 @@ require "fluent/test"
 require "fluent/test/driver/input"
 require "fluent/test/helpers"
 
+require "test/unit/rr"
+
 Test::Unit::TestCase.include(Fluent::Test::Helpers)
 Test::Unit::TestCase.extend(Fluent::Test::Helpers)

data/test/plugin/test_in_http_pull.rb

@@ -1,13 +1,12 @@
 require "helper"
 require "fluent/plugin/in_http_pull.rb"
 
-require 'webrick'
+require 'ostruct'
 
 class HttpPullInputTest < Test::Unit::TestCase
-  PORT = rand(1000) + 3000
   TEST_INTERVAL_3_CONFIG = %[
     tag test
-    url http://127.0.0.1:#{PORT}
+    url http://127.0.0.1
 
     interval 3
     status_only true
@@ -15,61 +14,72 @@ class HttpPullInputTest < Test::Unit::TestCase
 
   TEST_INTERVAL_5_CONFIG = %[
     tag test
-    url http://127.0.0.1:#{PORT}
+    url http://127.0.0.1
 
     interval 5
   ]
 
-  @test_server = nil
-
   setup do
     Fluent::Test.setup
+  end
 
-    Thread.new do
-      @test_server = WEBrick::HTTPServer.new :Port => PORT
-      @test_server.mount_proc '/' do |req, res|
-        res.status = 200
-        res['Content-Type'] = 'application/json'
-        res.body = '{"status":"OK"}'
+  sub_test_case "success case with status only" do
+    setup do
+      mock(RestClient).get("http://127.0.0.1").times(2) do
+        OpenStruct.new({code: 200, body: '{"status": "OK"}'})
       end
-      @test_server.start
     end
-  end
 
-  teardown do
-    @test_server.shutdown
-    sleep 1
-  end
+    test 'interval 3 with status_only' do
+      d = create_driver TEST_INTERVAL_3_CONFIG
+      assert_equal "test", d.instance.tag
 
-  test 'interval 3 with status_only' do
-    d = create_driver TEST_INTERVAL_3_CONFIG
-    assert_equal "test", d.instance.tag
+      d.run(timeout: 5) do
+        sleep 5
+      end
+      assert_equal 2, d.events.size
 
-    d.run(timeout: 8) do
-      sleep 8
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"url"=>"http://127.0.0.1","status"=>200}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
     end
-    assert_equal 3, d.events.size
 
-    d.events.each do |tag, time, record|
-      assert_equal("test", tag)
-      assert_equal({"url"=>"http://127.0.0.1:#{PORT}","status"=>200}, record)
-      assert(time.is_a?(Fluent::EventTime))
-    end
-  end
+    test 'interval 5' do
+      d = create_driver TEST_INTERVAL_5_CONFIG
+      assert_equal "test", d.instance.tag
 
-  test 'interval 5' do
-    d = create_driver TEST_INTERVAL_5_CONFIG
-    assert_equal "test", d.instance.tag
+      d.run(timeout: 7) do
+        sleep 7
+      end
+      assert_equal 2, d.events.size
 
-    d.run(timeout: 7) do
-      sleep 7
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal({"url"=>"http://127.0.0.1","status"=>200, "message"=>{"status"=>"OK"}}, record)
+        assert(time.is_a?(Fluent::EventTime))
+      end
     end
-    assert_equal 2, d.events.size
+  end
 
-    d.events.each do |tag, time, record|
-      assert_equal("test", tag)
-      assert_equal({"url"=>"http://127.0.0.1:#{PORT}","status"=>200, "message"=>{"status"=>"OK"}}, record)
-      assert(time.is_a?(Fluent::EventTime))
+  sub_test_case "fail when remote down" do
+    TEST_REFUSED_CONFIG = %[
+      tag test
+      url http://127.0.0.1:5927
+      interval 1
+    ]
+    test "connection refused by remote" do
+      d = create_driver TEST_REFUSED_CONFIG
+      d.run(timeout: 2) { sleep 3 }
+
+      assert_equal 3, d.events.size
+      d.events.each do |tag, time, record|
+        assert_equal("test", tag)
+        assert_equal("http://127.0.0.1:5927", record["url"])
+        assert_equal(0, record["status"])
+        assert(time.is_a?(Fluent::EventTime))
+      end
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: fluent-plugin-http-pull
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0.pre
 platform: ruby
 authors:
 - filepang
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-05-17 00:00:00.000000000 Z
+date: 2017-08-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rake
@@ -38,6 +38,26 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: test-unit-rr
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.0.2
 - !ruby/object:Gem::Dependency
   name: fluentd
   requirement: !ruby/object:Gem::Requirement
@@ -109,9 +129,9 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ">="
+  - - ">"
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 1.3.1
 requirements: []
 rubyforge_project: 
 rubygems_version: 2.5.2