observed-fluentd 0.1.0

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Mzk0ZDk4MjcyODVhNmQzMTM5ZDgzN2E1YzE1YTdiMWE0M2NlZjA1ZQ==
+  data.tar.gz: !binary |-
+    ZmU4Mzg5OThlZDU1MWZjYTJhMDcyMThmODY2NDFiZGY5MjI0MGE3Zg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    ZTAzZjY2MDBlYmUxYjYxNDU1YWNkYzFjMmEyMDc3OGUwYTQwZTk5ZDdjYzU3
+    N2VmZDEwYjQ1OTQ5YzIxZDBkYTRiMzU3YjI1YjdjODE5ODdhZTc0OTFiNzA2
+    NGFkNzhkZDYzNDM5NjUxNjJkZTJlNjZiZGVlMWFiMGVhYjllZjY=
+  data.tar.gz: !binary |-
+    ODgwNDNhNWQ3Yzc0YTIxY2EzYzZiMDdiZjY0OGY5ZWYwZTY0NDgyMTI5NjMz
+    NmQxMTUyZjZkMjY3MjkyMzA1YTlhNDA1ZWFmZmZhNjA4NjA4YTYwYzMyMDVj
+    OGI4YjU4ZmNkM2FmOTFmYTk2OWI3ODcyZDVkYjE1ODM3ODExNmM=

data/.gitignore

@@ -0,0 +1,20 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp
+*~
+*.log
+bin/

data/Gemfile

@@ -0,0 +1,12 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in observed-fluentd.gemspec
+gemspec
+
+gem 'clockwork', :group => :test
+gem 'fluentd', :group => :test
+gem 'cucumber', :group => :test
+gem 'aruba', :group => :test
+gem 'observed', :group => :test
+gem 'observed-http', :group => :test
+gem 'observed-clockwork', :group => :test

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 GREE, Inc.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,99 @@
+# Observed::Fluentd
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'observed-fluentd'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install observed-fluentd
+
+Or `git clone` the sources and install by using rake:
+
+    $ rake install
+
+## Usage
+
+First of all, we need to install the fluentd to which we send observed data:
+
+    $ gem install fluentd
+
+Then we should configure Fluentd to see data which will be sent from observed-fluentd:
+
+    $ fluentd --setup fluent.d
+    Installed fluent.d/fluent.conf.
+
+Ensure that the following 2 parts exist in the generated fluent.conf:
+
+     <source>
+       type forward
+     </source>
+
+    <match debug.**>
+      type stdout
+    </match>
+
+Now we can run Fluentd:
+
+    $ fluentd -c fluent.d/fluent.conf
+
+Install observed-fluentd:
+
+    $ gem build observed-fluentd.gemspec
+    $ gem install observed-fluentd
+    $ gem install observed-http
+    $ gem install observed-clockwork
+
+Create the observed.conf like:
+
+    require 'observed/builtin_plugins'
+    require 'observed/http'
+    require 'observed/fluentd'
+
+    observe 'myservice', {
+      plugin: 'http',
+      method: 'get',
+      url: 'http://localhost/',
+      timeout_in_milliseconds: 3000,
+    }
+
+    match /myservice.*/, {
+      plugin: 'fluentd',
+      host: 'localhost',
+      port: 24224,
+      tag: 'debug.myservice'
+    }
+
+Run clockwork and trigger observed:
+
+    $ cd observe.d/
+    $ clockwork clockwork.rb
+
+Find data sent from Observed in the Fluentd output:
+
+    2013-11-01T07:55:09Z	debug.myservice	{"status":"success","result":"Get http://localhost/","elapsed_time":0.013686}
+
+Now you can do anything utilizing Fluent's rich features and plugins.
+
+## Developping
+
+To run cucumber tests, execute:
+
+   $ bundle install --binstubs
+   $ bin/cucumber features
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/features/plugin.feature

@@ -0,0 +1,61 @@
+Feature: Receives Observed's input and send it to Fluentd
+
+  In order to pass data from Observed to Fluentd
+
+  I want to configure Observed to use observed-fluentd plugin
+
+  Scenario: Write configuration files for Observed and Fluentd, then run Fluentd, and then run Observed
+    Given a file named "test.rb" with:
+    """
+    require 'observed'
+    require 'observed/http'
+    require 'observed/fluentd'
+
+    include Observed
+
+    observe 'myservice', via: 'http', with: {
+      method: 'get',
+      url: 'http://google.com/',
+      timeout_in_milliseconds: 3000,
+    }
+
+    report /myservice.*/, via: 'fluentd', with: {
+      host: 'localhost',
+      port: 24224,
+      tag: 'debug.myservice'
+    }
+
+    run 'myservice'
+    """
+    Given a file named "fluent.conf" with:
+    """
+    <source>
+      type forward
+    </source>
+
+    <match debug.**>
+      type file
+      path fluent.out
+      time_slice_format foo
+      utc
+      flush_interval 1s
+    </match>
+    """
+    When I run `pwd`
+    When I run `fluentd -d pid_file -c fluent.conf`
+#    When I start my daemon with "fluentd -c tmp/aruba/fluent.conf"
+    When I run `sleep 3`
+    When I run `ruby test.rb`
+    #Then a daemon called "fluentd" should be running
+    When I run `sleep 3`
+    When I run the command "kill $(cat tmp/aruba/pid_file)"
+    When I run `cat fluent.out.foo_0.log`
+    Then the output should contain:
+    """
+    elapsed_time
+    """
+    Then the output should contain:
+    """
+    debug.myservice
+    """
+

data/features/support/env.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__) + '/../../lib')
+require 'aruba/cucumber'
+
+World(Aruba::Api)
+
+Before do
+  @aruba_timeout_seconds = 120
+end
+
+When /^I run the command "([^"]+)"$/ do |cmd|
+  system cmd
+end
+
+When /^I start my daemon with "([^"]*)"$/ do |cmd|
+  @root = Pathname.new(File.dirname(__FILE__)).parent.parent.expand_path
+  command = "#{@root.join('bin')}/#{cmd}"
+
+  puts "In the working directory: #{Dir.pwd}"
+  puts "Running #{command}"
+
+  @pipe = IO.popen(command, "r")
+  sleep 2 # so the daemon has a chance to boot
+
+  # clean up the daemon when the tests finish
+  at_exit do
+    Process.kill("KILL", @pipe.pid)
+  end
+end
+
+Then /^a daemon called "([^"]*)" should be running$/ do |daemon|
+  `ps -eo command |grep #{daemon}`.size.should > 0
+end

data/fluent.d/fluent.conf

@@ -0,0 +1,93 @@
+
+## built-in TCP input
+## $ echo <json> | fluent-cat <tag>
+<source>
+  type forward
+</source>
+
+## built-in UNIX socket input
+#<source>
+#  type unix
+#</source>
+
+# HTTP input
+# http://localhost:8888/<tag>?json=<json>
+<source>
+  type http
+  port 8888
+</source>
+
+## File input
+## read apache logs with tag=apache.access
+#<source>
+#  type tail
+#  format apache
+#  path /var/log/httpd-access.log
+#  tag apache.access
+#</source>
+
+# Listen HTTP for monitoring
+# http://localhost:24220/api/plugins
+# http://localhost:24220/api/plugins?type=TYPE
+# http://localhost:24220/api/plugins?tag=MYTAG
+<source>
+  type monitor_agent
+  port 24220
+</source>
+
+# Listen DRb for debug
+<source>
+  type debug_agent
+  port 24230
+</source>
+
+
+## match tag=apache.access and write to file
+#<match apache.access>
+#  type file
+#  path /var/log/fluent/access
+#</match>
+
+## match tag=debug.** and dump to console
+<match debug.**>
+  type stdout
+</match>
+
+# match tag=system.** and forward to another fluent server
+<match system.**>
+  type forward
+  host 192.168.0.11
+  <secondary>
+    host 192.168.0.12
+  </secondary>
+</match>
+
+## match tag=myapp.** and forward and write to file
+#<match myapp.**>
+#  type copy
+#  <store>
+#    type forward
+#    host 192.168.0.13
+#    buffer_type file
+#    buffer_path /var/log/fluent/myapp-forward
+#    retry_limit 50
+#    flush_interval 10s
+#  </store>
+#  <store>
+#    type file
+#    path /var/log/fluent/myapp
+#  </store>
+#</match>
+
+## match fluent's internal events
+#<match fluent.**>
+#  type null
+#</match>
+
+## match not matched logs and write to file
+#<match **>
+#  type file
+#  path /var/log/fluent/else
+#  compress gz
+#</match>
+

data/fluentd.pid

@@ -0,0 +1 @@
+3921

data/lib/observed/fluentd/version.rb

@@ -0,0 +1,5 @@
+module Observed
+  class Fluentd
+    VERSION = "0.1.0"
+  end
+end

data/lib/observed/fluentd.rb

@@ -0,0 +1,30 @@
+require 'observed/fluentd/version'
+require 'observed/reporter'
+require 'observed/reporter/regexp_matching'
+require 'fluent-logger'
+
+module Observed
+  module Plugins
+    class Fluentd < Observed::Reporter
+
+      include Observed::Reporter::RegexpMatching
+
+      plugin_name 'fluentd'
+
+      attribute :tag
+      attribute :host
+      attribute :port, default: 24224
+      attribute :transform, default: ->(data){ data }
+
+      def report(tag, time, data)
+        fluent_logger.post(self.tag, transform.call(data))
+      end
+
+      private
+
+      def fluent_logger
+        @fluent_logger ||= Fluent::Logger::FluentLogger.new(nil, host: host, port: port)
+      end
+    end
+  end
+end

data/observe.d/clockwork.rb

@@ -0,0 +1,9 @@
+require 'clockwork'
+require 'observed/clockwork'
+
+include Clockwork
+include Observed::Clockwork
+
+observed :config_file => 'observed.conf'
+
+every(10.seconds, 'myservice')

data/observe.d/observed.conf

@@ -0,0 +1,17 @@
+require 'observed/builtin_plugins'
+require 'observed/http'
+require 'observed/fluentd'
+
+observe 'myservice', {
+  plugin: 'http',
+  method: 'get',
+  url: 'http://localhost/',
+  timeout_in_milliseconds: 3000,
+}
+
+match /myservice.*/, {
+  plugin: 'fluentd',
+  host: 'localhost',
+  port: 24224,
+  tag: 'debug.myservice'
+}

data/observed-fluentd.gemspec

@@ -0,0 +1,26 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'observed/fluentd/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'observed-fluentd'
+  spec.version       = Observed::Fluentd::VERSION
+  spec.authors       = ['KUOKA Yusuke']
+  spec.email         = %w(yusuke.kuoka@gree.net)
+  spec.description   = %q{Observed Fluentd Output Plugin}
+  spec.summary       = %q{observed-fluentd is an Observed output plugin for sending observed data to Fluentd}
+  spec.homepage      = ''
+  spec.license       = 'MIT'
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = %w(lib)
+
+  spec.add_dependency 'fluent-logger', '~> 0.4.6'
+
+  spec.add_development_dependency 'bundler', '~> 1.3'
+  spec.add_development_dependency 'rake'
+  spec.add_development_dependency 'rspec'
+end

metadata

@@ -0,0 +1,117 @@
+--- !ruby/object:Gem::Specification
+name: observed-fluentd
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- KUOKA Yusuke
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-11-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: fluent-logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: 0.4.6
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Observed Fluentd Output Plugin
+email:
+- yusuke.kuoka@gree.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- features/plugin.feature
+- features/support/env.rb
+- fluent.d/fluent.conf
+- fluentd.pid
+- lib/observed/fluentd.rb
+- lib/observed/fluentd/version.rb
+- observe.d/clockwork.rb
+- observe.d/observed.conf
+- observed-fluentd.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.5
+signing_key: 
+specification_version: 4
+summary: observed-fluentd is an Observed output plugin for sending observed data to
+  Fluentd
+test_files:
+- features/plugin.feature
+- features/support/env.rb