fluent-plugin-exec_cron 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: cbbcf3f7345626fd259b551bebfca72f62a994cd
+  data.tar.gz: 24ca495172005d6079578070c02ad81a371023ad
+SHA512:
+  metadata.gz: f4f551d1ef67f94df3df94f9b1423c1157cd1c9e4e2c57c4b8d1f1ab1e4e2715442075e6158076426eca32688aab17b6132f4166d2d079eae166e10a46982a3b
+  data.tar.gz: 98b8abdcf12ef59e76b02b2b9c01b609eaeee4016cf55a89e6a10172d9d2c925fd2f96b1c069f54243c45f9227a044f72b0da2289c629f419114aa28a579a6ad

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,9 @@
+Gemfile.lock
+doc/
+pkg/
+**/*.gem
+.yardoc/
+.bundle/
+vendor/bundle/
+.tags
+tmp/

data/.travis.yml

@@ -0,0 +1,15 @@
+language: ruby
+
+rvm:
+  - 1.9.3
+  - 2.0.0
+  - 2.1
+
+os:
+  - linux
+  - osx
+
+gemfile:
+  - Gemfile
+
+script: 'bundle exec rake test'

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "fluent-plugin-exec_cron Documentation" --protected

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/README.md

@@ -0,0 +1,79 @@
+# fluent-plugin-exec_cron [![Build Status](https://secure.travis-ci.org/toyama0919/fluent-plugin-exec_cron.png?branch=master)](http://travis-ci.org/toyama0919/fluent-plugin-exec_cron)
+
+executes external programs with cron syntax.
+
+## Examples
+
+### minutely
+```
+<source>
+  type exec_cron
+  tag exec_cron.example
+  command echo '{"a":"a"}'
+  format json
+  cron * * * * *
+  graceful_shutdown false
+</source>
+
+<match exec_cron.example>
+  type stdout
+</match>
+```
+
+#### output
+```
+2015-05-27 13:07:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:08:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:09:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:10:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:11:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:12:00 +0900 exec_cron.example: {"a":"a"}
+```
+
+### hourly
+```
+<source>
+  type exec_cron
+  tag exec_cron.example
+  command echo '{"a":"a"}'
+  format json
+  cron 0 * * * *
+  graceful_shutdown true
+</source>
+
+<match exec_cron.example>
+  type stdout
+</match>
+```
+
+#### output
+```
+2015-05-27 12:00:00 +0900 exec_cron.example: {"a":"a"}
+2015-05-27 13:00:00 +0900 exec_cron.example: {"a":"a"}
+```
+
+
+## Installation
+```
+sudo td-agent-gem install fluent-plugin-exec_cron
+```
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Information
+
+* [Homepage](https://github.com/toyama0919/fluent-plugin-exec_cron)
+* [Issues](https://github.com/toyama0919/fluent-plugin-exec_cron/issues)
+* [Documentation](http://rubydoc.info/gems/fluent-plugin-exec_cron/frames)
+* [Email](mailto:toyama0919@gmail.com)
+
+## Copyright
+
+Copyright (c) 2015 Hiroshi Toyama
+

data/Rakefile

@@ -0,0 +1,11 @@
+require 'bundler/gem_tasks'
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.pattern = 'test/**/test_*.rb'
+  test.verbose = true
+end
+
+task :default => :test
+

data/exsample/exsample.conf

@@ -0,0 +1,18 @@
+<source>
+  type forward
+  port 24224
+</source>
+
+<source>
+  type exec_cron
+  tag exec_cron.example
+  command echo '{"a":"a"}'
+  format json
+  cron * * * * *
+  graceful_shutdown false
+</source>
+
+<match exec_cron.exsample>
+  type stdout
+</match>
+

data/fluent-plugin-exec_cron.gemspec

@@ -0,0 +1,26 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-plugin-exec_cron"
+  gem.version       = "0.0.2"
+  gem.summary       = %q{executes external programs with cron syntax.}
+  gem.description   = %q{executes external programs with cron syntax.}
+  gem.license       = "MIT"
+  gem.authors       = ["Hiroshi Toyama"]
+  gem.email         = "toyama0919@gmail.com"
+  gem.homepage      = "https://github.com/toyama0919/fluent-plugin-exec_cron"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_runtime_dependency "parse-cron"
+  gem.add_development_dependency 'bundler', '~> 1.7.2'
+  gem.add_development_dependency 'fluentd'
+  gem.add_development_dependency 'pry', '~> 0.10.1'
+  gem.add_development_dependency 'rake', '~> 10.3.2'
+  gem.add_development_dependency 'rubocop', '~> 0.24.1'
+  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  gem.add_development_dependency 'yard', '~> 0.8'
+end

data/lib/fluent/plugin/in_exec_cron.rb

@@ -0,0 +1,136 @@
+module Fluent
+  class ExecCronInput < Input
+    Plugin.register_input('exec_cron', self)
+
+    def initialize
+      super
+      require 'fluent/plugin/exec_util'
+      require 'fluent/timezone'
+      require 'parse-cron'
+      require 'erb'
+    end
+
+    SUPPORTED_FORMAT = {
+      'tsv' => :tsv,
+      'json' => :json,
+      'msgpack' => :msgpack,
+    }
+
+    unless method_defined?(:router)
+      define_method("router") { Fluent::Engine }
+    end
+
+    config_param :command, :string
+    config_param :format, :default => :tsv do |val|
+      f = SUPPORTED_FORMAT[val]
+      raise ConfigError, "Unsupported format '#{val}'" unless f
+      f
+    end
+    config_param :keys, :default => [] do |val|
+      val.split(',')
+    end
+    config_param :tag, :string, :default => nil
+    config_param :tag_key, :string, :default => nil
+    config_param :time_key, :string, :default => nil
+    config_param :time_format, :string, :default => nil
+    config_param :run_interval, :time, :default => nil
+    config_param :graceful_shutdown, :bool, :default => false
+    config_param :cron, :string
+
+    def configure(conf)
+      super
+
+      if localtime = conf['localtime']
+        @localtime = true
+      elsif utc = conf['utc']
+        @localtime = false
+      end
+
+      if conf['timezone']
+        @timezone = conf['timezone']
+        Fluent::Timezone.validate!(@timezone)
+      end
+
+      if !@tag && !@tag_key
+        raise ConfigError, "'tag' or 'tag_key' option is required on exec input"
+      end
+
+      if @time_key
+        if @time_format
+          f = @time_format
+          @time_parse_proc = Proc.new {|str| Time.strptime(str, f).to_i }
+        else
+          @time_parse_proc = Proc.new {|str| str.to_i }
+        end
+      end
+
+      case @format
+      when :tsv
+        if @keys.empty?
+          raise ConfigError, "keys option is required on exec input for tsv format"
+        end
+        @parser = ExecUtil::TSVParser.new(@keys, method(:on_message))
+      when :json
+        @parser = ExecUtil::JSONParser.new(method(:on_message))
+      when :msgpack
+        @parser = ExecUtil::MessagePackParser.new(method(:on_message))
+      end
+
+      begin
+        @cron_parser = CronParser.new(@cron)
+      rescue => e
+        raise ConfigError, "invalid cron expression. [#{@cron}]"
+      end
+      @command = ERB.new(@command.gsub(/\$\{([^}]+)\}/, '<%= \1 %>'))
+    end
+
+    def start
+      @finished = false
+      @thread = Thread.new(&method(:run_periodic))
+    end
+
+    def shutdown
+      @finished = true
+      if @graceful_shutdown
+        @thread.join
+      else
+        Thread.kill(@thread)
+      end
+    end
+
+    def run_periodic
+      until @finished
+        begin
+          secs = @cron_parser.next(Time.now) - Time.now
+          sleep secs
+          io = IO.popen(@command.result(binding), "r")
+          @parser.call(io)
+          Process.waitpid(io.pid)
+        rescue
+          log.error "exec failed to run or shutdown child process", :error => $!.to_s, :error_class => $!.class.to_s
+          log.warn_backtrace $!.backtrace
+        end
+      end
+    end
+
+    private
+
+    def on_message(record)
+      if val = record.delete(@tag_key)
+        tag = val
+      else
+        tag = @tag
+      end
+
+      if val = record.delete(@time_key)
+        time = @time_parse_proc.call(val)
+      else
+        time = Engine.now
+      end
+
+      router.emit(tag, time, record)
+    rescue => e
+      log.error "exec failed to emit", :error => e.to_s, :error_class => e.class.to_s, :tag => tag, :record => Yajl.dump(record)
+    end
+  end
+end

data/test/helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'bundler'
+begin
+  Bundler.setup(:default, :development)
+rescue Bundler::BundlerError => e
+  $stderr.puts e.message
+  $stderr.puts "Run `bundle install` to install missing gems"
+  exit e.status_code
+end
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+require 'fluent/test'
+require 'fluent/plugin/in_exec_cron'
+
+class Test::Unit::TestCase
+end

data/test/plugin/test_in_exec_cron.rb

@@ -0,0 +1,75 @@
+require 'helper'
+
+class ExecCronInputTest < Test::Unit::TestCase
+  def setup
+    Fluent::Test.setup
+  end
+
+  def create_driver(conf)
+    Fluent::Test::InputTestDriver.new(Fluent::ExecCronInput).configure(conf)
+  end
+
+  def test_configure_full
+    d = create_driver %q{
+      type exec_cron
+      tag exec_cron.example
+      command echo '{"a":"a"}'
+      format json
+      cron * * * * *
+      graceful_shutdown false
+    }
+
+    assert_equal 'exec_cron.example', d.instance.tag
+  end
+
+  def test_configure_error_when_config_is_empty
+    assert_raise(Fluent::ConfigError) do
+      create_driver ''
+    end
+  end
+
+  def test_configure_error_cron_syntax
+    assert_raise(Fluent::ConfigError) do
+      create_driver %q{
+        tag exec_cron.example
+        command echo '{"a":"a"}'
+        format json
+        cron hogehoge
+        graceful_shutdown false
+      }
+    end
+  end
+
+  def test_configure_placeholder
+    d = create_driver %q{
+      type exec_cron
+      tag exec_cron.example
+      command echo '{"a":"${(Date.new(2015, 1, 1) - 1).strftime('%Y/%m/%d')}"}'
+      format json
+      cron * * * * *
+      graceful_shutdown false
+    }
+    assert_equal %Q{echo '{"a":"2014/12/31"}'}, d.instance.command.result(binding)
+  end
+
+  def test_emit
+    d = create_driver %q{
+      tag exec_cron.example
+      command echo '{"a":"a", "time":"2011-01-02 13:14:15"}'
+      format json
+      time_key time
+      time_format %Y-%m-%d %H:%M:%S
+      cron * * * * *
+      graceful_shutdown false
+    }
+
+    d.run do
+      sleep 60
+    end
+
+    emits = d.emits
+    assert_equal true, emits.length > 0
+    assert_equal ["exec_cron.example", Time.parse("2011-01-02 13:14:15").to_i, {"a"=>"a"}], emits[0]
+  end
+
+end

metadata

@@ -0,0 +1,170 @@
+--- !ruby/object:Gem::Specification
+name: fluent-plugin-exec_cron
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-06-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parse-cron
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.1
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: rubocop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.24.1
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: executes external programs with cron syntax.
+email: toyama0919@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".travis.yml"
+- ".yardopts"
+- Gemfile
+- README.md
+- Rakefile
+- exsample/exsample.conf
+- fluent-plugin-exec_cron.gemspec
+- lib/fluent/plugin/in_exec_cron.rb
+- test/helper.rb
+- test/plugin/test_in_exec_cron.rb
+homepage: https://github.com/toyama0919/fluent-plugin-exec_cron
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.2
+signing_key: 
+specification_version: 4
+summary: executes external programs with cron syntax.
+test_files:
+- test/helper.rb
+- test/plugin/test_in_exec_cron.rb
+has_rdoc: