fluent-plugin-time_parser 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    Yjk2ZDdlYTEwNDRjM2EyNDY3MmU3N2RhYjc2YjI3MmI0NWE3Nzk3Mw==
+  data.tar.gz: !binary |-
+    YzE2YzcxM2IxYTg2NjVhYjgzYmE2MjY2OGJlNDZmYzlkYTQ3NDA3Zg==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MDIwMjU5NzdiZjFjZDczZWU5ODU4ZTc4ZTkwNjdhNDI1MDlmNGQwNzA0YTky
+    MWM2ODIyZDZkMjE4NDE5ZjRlNDc0NDNkOWU0YTgwZmNlOGYyODA4ZWFmMzY1
+    MmU4OGI4OGE0ZmE4ODI3NGVkODgyNWM4YzI5OGM0Y2ZjMWU5OWQ=
+  data.tar.gz: !binary |-
+    ZWE1NmFmYjFmMWE4MTU4MTE4YzE3YTlmMTQyMjYxZDc4OGYyMjEwM2YwMTY5
+    MmQ1ZDY2MjRkNGZiY2UwOTNjYmUzMzBjZGJkNWUyYTNjMjhjNTUxMjAwYzIz
+    ZmI5MTJmNDM4MzgyODgzZmE4NDFjMGEyMDRmZmZkNzdmZDI3NDg=

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,2 @@
+Copyright (c) 2012 Kentaro Kuribayashi
+Apache License, Version 2.0

data/README.md

@@ -0,0 +1,22 @@
+# fluent-plugin-time_parser
+
+## Component
+
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Added some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request
+
+## Copyright
+
+### Copyright
+
+Copyright (c) 2013- Carlos Donderis (@CaDs)
+
+### License
+
+Apache License, Version 2.0

data/Rakefile

@@ -0,0 +1,9 @@
+#!/usr/bin/env rake
+require "bundler/gem_tasks"
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end

data/fluent-plugin-time_parser.gemspec

@@ -0,0 +1,18 @@
+Gem::Specification.new do |gem|
+  gem.name          = 'fluent-plugin-time_parser'
+  gem.version       = '0.0.1'
+  gem.authors       = ['Carlos Donderis']
+  gem.email         = ['cdonderis@gmail.com']
+  gem.homepage      = 'http://github.com/cads/fluent-plugin-time_parser'
+  gem.description   = %q{Fluentd plugin to parse the time parameter.}
+  gem.summary       = %q{Fluentd plugin to parse the time parameter.}
+
+  gem.files         = `git ls-files`.split($\)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'fluentd'
+  gem.add_runtime_dependency     'fluentd'
+end

data/lib/fluent/plugin/out_time_parser.rb

@@ -0,0 +1,62 @@
+require 'time'
+require 'tzinfo'
+
+module Fluent
+  class TimeParserOutput < Output
+    include Fluent::HandleTagNameMixin
+    Fluent::Plugin.register_output('time_parser', self)
+
+    config_param :key, :string, :default => 'time'
+    config_param :time_zone, :string
+
+    def configure(conf)
+      super
+      if (
+          !remove_tag_prefix &&
+          !remove_tag_suffix &&
+          !add_tag_prefix    &&
+          !add_tag_suffix
+      )
+        raise ConfigError, "out_extract_query_params: At least one of remove_tag_prefix/remove_tag_suffix/add_tag_prefix/add_tag_suffix is required to be set."
+        raise ConfigError, "You must specify a time zone" unless time_zone
+      end
+    end
+
+    def start
+      super
+    end
+
+    def shutdown
+      super
+    end
+
+    def emit(tag, es, chain)
+      es.each {|time,record|
+        t = tag.dup
+        filter_record(t, time, record)
+        Engine.emit(t, time, record)
+      }
+      chain.next
+    end
+
+    def filter_record(tag, time, record)
+      begin
+        original_time = Time.parse(record[key])
+        tz = TZInfo::Timezone.get(time_zone.capitalize)
+
+        converted_time = tz.utc_to_local(original_time)
+
+        date = converted_time.to_date.to_s
+        hour = converted_time.hour.to_s
+        record['date'] = date
+        record['hour'] = hour
+      rescue ArgumentError => error
+        $log.warn("out_extract_query_params: #{error.message}")
+      rescue TZInfo::InvalidTimezoneIdentifier
+        $log.warn("Timezone Not Valid, please refer to http://tzinfo.rubyforge.org/doc/classes/TZInfo/Timezone.html for valid timezones")
+      end
+
+      super(tag, time, record)
+    end
+  end
+end

data/test/plugin/test_out_time_parser.rb

@@ -0,0 +1,141 @@
+# -*- encoding: utf-8 -*-
+require 'test_helper'
+require 'time'
+require 'tzinfo'
+require 'debugger'
+
+class TimeParserOutputTest < Test::Unit::TestCase
+
+  TIME = "2013-04-14T06:14:36Z"
+
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf, tag = 'test')
+    Fluent::Test::OutputTestDriver.new(
+      Fluent::TimeParserOutput, tag
+    ).configure(conf)
+  end
+
+  def test_configure
+    d = create_driver(%[
+      key            test
+      add_tag_prefix extracted.
+      time_zone      Tokyo
+    ])
+    assert_equal 'test', d.instance.key
+    assert_equal 'extracted.', d.instance.add_tag_prefix
+    assert_equal 'Tokyo',   d.instance.time_zone
+
+    #Default Key
+    d = create_driver(%[
+      add_tag_prefix extracted.
+      time_zone      Tokyo
+    ])
+    assert_equal 'time', d.instance.key
+    assert_equal 'extracted.', d.instance.add_tag_prefix
+    assert_equal 'Tokyo',   d.instance.time_zone
+
+    #No Prefix
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        time_zone      Tokyo
+      ])
+    end
+    #No TimeZone
+    assert_raise(Fluent::ConfigError) do
+      create_driver(%[
+        key            test
+        add_tag_prefix extracted.
+      ])
+    end
+  end
+
+  def test_filter_record
+    d = create_driver(%[
+      key            time
+      add_tag_prefix extracted.
+      time_zone      Japan
+    ])
+    tag    = 'test'
+    record = {'time' => TIME}
+    d.instance.filter_record('test', Time.now, record)
+    tz = TZInfo::Timezone.get('Japan')
+    converted_time = tz.utc_to_local(Time.parse(TIME))
+
+    assert_equal record['date'], converted_time.to_date.to_s
+    assert_equal record['hour'], converted_time.hour.to_s
+  end
+
+  def test_filter_record_bad_parameters
+    d = create_driver(%[
+      key            time
+      add_tag_prefix extracted.
+      time_zone      myPlace
+    ])
+    tag    = 'test'
+    record = {'time' => TIME}
+
+    d.instance.filter_record('test', Time.now, record)
+    assert_equal record['date'], nil
+    assert_equal record['hour'], nil
+
+    record = {'time' => "this is not a date"}
+    d.instance.filter_record('test', Time.now, record)
+    assert_equal record['date'], nil
+    assert_equal record['hour'], nil
+  end
+
+  def test_emit
+    d = create_driver(%[
+      key            time
+      add_tag_prefix extracted.
+      time_zone      Japan
+    ])
+
+    d.run { d.emit('time' => TIME) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'extracted.test', emits[0][0]
+    assert_equal TIME, emits[0][2]['time']
+  end
+
+  def test_emit_multi
+    d = create_driver(%[
+      key            time
+      add_tag_prefix extracted.
+      time_zone      Japan
+    ])
+
+    d.run do
+      d.emit('time' => TIME)
+      d.emit('time' => TIME)
+      d.emit('time' => TIME)
+    end
+    emits = d.emits
+
+    assert_equal 3, emits.count
+    0.upto(2) do |i|
+      assert_equal 'extracted.test', emits[i][0]
+      assert_equal TIME, emits[i][2]['time']
+    end
+  end
+
+  def test_emit_with_invalid_time
+    d = create_driver(%[
+      key            time
+      add_tag_prefix extracted.
+      time_zone      Japan
+    ])
+    wrong_time = 'wrong time'
+    d.run { d.emit('time' => wrong_time) }
+    emits = d.emits
+
+    assert_equal 1, emits.count
+    assert_equal 'extracted.test', emits[0][0]
+    assert_equal wrong_time, emits[0][2]['time']
+  end
+
+end

data/test/test_helper.rb

@@ -0,0 +1,21 @@
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+
+require 'fluent/test'
+
+unless ENV.has_key?('VERBOSE')
+  nulllogger = Object.new
+  nulllogger.instance_eval {|obj|
+    def method_missing(method, *args)
+      # pass
+    end
+  }
+  $log = nulllogger
+end
+
+require 'fluent/plugin/out_time_parser'
+
+class Test::Unit::TestCase
+end

metadata

@@ -0,0 +1,96 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-time_parser
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Carlos Donderis
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-04-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Fluentd plugin to parse the time parameter.
+email:
+- cdonderis@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- fluent-plugin-time_parser.gemspec
+- lib/fluent/.DS_Store
+- lib/fluent/plugin/out_time_parser.rb
+- test/plugin/test_out_time_parser.rb
+- test/test_helper.rb
+homepage: http://github.com/cads/fluent-plugin-time_parser
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Fluentd plugin to parse the time parameter.
+test_files:
+- test/plugin/test_out_time_parser.rb
+- test/test_helper.rb