fluent-client 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: e2e494e8e7e21884073391ee32c0b15a788401a7
+  data.tar.gz: 43a49b73cf01311ea1b352e32e5d4c7443317407
+SHA512:
+  metadata.gz: 685a9d71440c5c5bc74671433b2e44b0afba112e3f95be46d905ec4355feed3ad19c69d29e12e067c14472034b182d2489b630dd984a4afdf79e097a9fc59f72
+  data.tar.gz: 805b700ac45ee053d510523850a5c6c4a8ea756fb51ef1a2308ff07d1e428166d0bb320652f6eaa13c84b4fb622e8a5611169d5e135e32278ffde446505af8dc

data/.document

@@ -0,0 +1,3 @@
+- 
+ChangeLog.md
+LICENSE.txt

data/.gitignore

@@ -0,0 +1,7 @@
+Gemfile.lock
+doc/
+pkg/
+**/*.gem
+.yardoc/
+.bundle/
+vendor/bundle/

data/.rspec

@@ -0,0 +1 @@
+--colour --format documentation

data/.yardopts

@@ -0,0 +1 @@
+--markup markdown --title "fluent-client Documentation" --protected

data/ChangeLog.md

@@ -0,0 +1,4 @@
+### 0.1.0 / 2014-09-14
+
+* Initial release:
+

data/Gemfile

@@ -0,0 +1,7 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'kramdown'
+end

data/LICENSE.txt

@@ -0,0 +1,20 @@
+Copyright (c) 2014 Hiroshi Toyama
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,98 @@
+# fluent-client
+
+fluentd command line utility
+
+## Examples
+
+### simple post to fluentd
+
+    $ fluent-client --post sometag --data id:1 name:toyama
+    #=> 2014-09-14 17:43:40 +0900 sometag: {"id":"1","name":"toyama"}
+
+### simple post to fluentd other host
+
+    $ fluent-client --post sometag --data id:1 name:toyama -h aggregate-host -p 24223
+
+### post stdin json
+
+    $ echo '[{"hoge":"fuga"},{"hoge":"fuga2"}]' | fluent-client -j sometag
+    #=> 2014-09-14 22:30:26 +0900 sometag: {"hoge":"fuga"}
+    #=> 2014-09-14 22:30:26 +0900 sometag: {"hoge":"fuga2"}
+
+### post stdin file with regular expression
+
+    $ cat server.csv
+    i-xxxxxxx web01 running
+    i-xxxxxxx web02 running
+    i-xxxxxxx web03 running
+    i-xxxxxxx web04 running
+    i-xxxxxxx batch01 running
+
+    $ cat server.txt | fluent-client post_parse_text sometag --format "/^(?<instance_id>[^ ]*) (?<name>[^ ]*) (?<state>[^ ]*)$/"
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web01", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web02", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web03", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web04", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"batch01", "state"=>"running"}
+
+※ multiline not support
+
+### post stdin csv file
+
+    $ cat server.csv
+    i-xxxxxxx,web01,running
+    i-xxxxxxx,web02,running
+    i-xxxxxxx,web03,running
+    i-xxxxxxx,web04,running
+    i-xxxxxxx,batch01,running
+
+    $ cat server.csv | fluent-client post_parse_text sometag --format csv --keys instance_id,name,state
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web01", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web02", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web03", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"web04", "state"=>"running"}
+    #=> {"instance_id"=>"i-xxxxxxx", "name"=>"batch01", "state"=>"running"}
+
+
+
+## default values
+
+default host: localhost
+
+default port: 24224
+
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'fluent-client'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install fluent-client
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new [Pull Request](../../pull/new/master)
+
+## Information
+
+* [Homepage](https://github.com/toyama0919/fluent-client)
+* [Issues](https://github.com/toyama0919/fluent-client/issues)
+* [Documentation](http://rubydoc.info/gems/fluent-client/frames)
+* [Email](mailto:toyama0919@gmail.com)
+
+## Copyright
+
+Copyright (c) 2014 Hiroshi Toyama
+
+See [LICENSE.txt](../LICENSE.txt) for details.

data/Rakefile

@@ -0,0 +1,34 @@
+# encoding: utf-8
+
+require 'rubygems'
+
+begin
+  require 'bundler'
+rescue LoadError => e
+  warn e.message
+  warn "Run `gem install bundler` to install Bundler."
+  exit -1
+end
+
+begin
+  Bundler.setup(:development)
+rescue Bundler::BundlerError => e
+  warn e.message
+  warn "Run `bundle install` to install missing gems."
+  exit e.status_code
+end
+
+require 'rake'
+
+require 'rubygems/tasks'
+Gem::Tasks.new
+
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new
+
+task :test    => :spec
+task :default => :spec
+
+require 'yard'
+YARD::Rake::YardocTask.new  
+task :doc => :yard

data/bin/fluent-client

@@ -0,0 +1,6 @@
+#!/usr/bin/env ruby
+
+require 'rubygems'
+require 'fluent/client'
+
+Fluent::Client::CLI.start

data/fluent-client.gemspec

@@ -0,0 +1,31 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path('../lib/fluent/client/version', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.name          = "fluent-client"
+  gem.version       = Fluent::Client::VERSION
+  gem.summary       = %q{fluentd command line utility}
+  gem.description   = %q{fluentd command line utility}
+  gem.license       = "MIT"
+  gem.authors       = ["Hiroshi Toyama"]
+  gem.email         = "toyama0919@gmail.com"
+  gem.homepage      = "https://github.com/toyama0919/fluent-client"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ['lib']
+
+  gem.add_dependency 'thor', '~> 0.19.1'
+  gem.add_dependency 'fluentd'
+  gem.add_dependency 'yajl-ruby'
+  gem.add_dependency 'fluent-logger'
+  gem.add_dependency 'activesupport'
+
+  gem.add_development_dependency 'bundler', '~> 1.7.2'
+  gem.add_development_dependency 'rake', '~> 10.3.2'
+  gem.add_development_dependency 'rspec', '~> 2.4'
+  gem.add_development_dependency 'rubygems-tasks', '~> 0.2'
+  gem.add_development_dependency 'yard', '~> 0.8'
+end

data/lib/fluent/client.rb

@@ -0,0 +1,3 @@
+require 'fluent/client/version'
+require 'fluent/client/core'
+require 'fluent/client/cli'

data/lib/fluent/client/cli.rb

@@ -0,0 +1,97 @@
+#! /usr/bin/env ruby
+# -*- encoding: UTF-8 -*-
+require 'thor'
+require 'yajl'
+require 'fluent-logger'
+require 'active_support/core_ext/string'
+require 'fluent/log'
+require 'fluent/config'
+require 'fluent/engine'
+require 'fluent/parser'
+
+module Fluent
+  module Client
+    class CLI < Thor
+      map '--post' => :post
+      map '-i' => :stdin
+      map '-l' => :stdin_line
+      map '-j' => :post_json
+
+      class_option :host, type: :string, default: 'localhost', aliases: '-h', desc: 'config file'
+      class_option :port, type: :numeric, default: 24_224, aliases: '-p', desc: 'config file'
+      def initialize(args = [], options = {}, config = {})
+        super(args, options, config)
+        @global_options = config[:shell].base.options
+        @logger = Fluent::Logger::FluentLogger.new(nil, host: @global_options['host'], port: @global_options['port'])
+        @core = Core.new
+      end
+
+      desc '--post', 'fluent-client --post sometag --data id:1 name:toyama'
+      option :data, type: :hash, desc: 'config file', required: true
+      option :time_key, type: :string, default: nil, desc: 'config file'
+      def post(tag)
+        data = options['data']
+        if options['time_key'].nil?
+          @logger.post(tag, data)
+        else
+          @logger.post_with_time(tag, options['data'], data[options['time_key']].to_time)
+        end
+      end
+
+      desc '-j [tag]', %(echo '{"hoge":"fuga"}' | fluent-client -j sometag)
+      def post_json(tag)
+        results = parse_json(STDIN.read)
+        results.each do |result|
+          @logger.post(tag, result)
+        end
+      end
+
+      desc '-i', 'echo -n hoge | fluent-client -i sometag'
+      def stdin(tag)
+        @logger.post(tag, message: STDIN.read)
+      end
+
+      desc '-l', 'cat somefile | fluent-client -l sometag'
+      def stdin_line(tag)
+        STDIN.read.lines.each do |line|
+          @logger.post(tag, message: line)
+        end
+      end
+
+      desc 'validate_parse [tag]',
+           'echo "2014-01-01 localhost test" | fluent-client validate_parse --format "/^(?<log_date>\d{4}-\d{2}-\d{2}) (?<host>[^ ]*) (?<huga>[^ ]*)$/"'
+      option :format, type: :string, default: nil, desc: 'format regexp', required: true
+      option :time_format, type: :string, default: nil, desc: 'time format'
+      option :keys, type: :string, default: nil, desc: 'time format'
+      def validate_parse
+        @core.parse_text(options['format'], options['time_format'], options['keys'])
+      end
+
+      desc 'post_parse_text [tag]',
+           'echo "2014-01-01 localhost test" | fluent-client post_parse_text sometag --format "/^(?<log_date>\d{4}-\d{2}-\d{2}) (?<host>[^ ]*) (?<huga>[^ ]*)$/"'
+      option :format, type: :string, default: nil, desc: 'format regexp', required: true
+      option :time_format, type: :string, default: nil, desc: 'time format'
+      option :keys, type: :string, default: nil, desc: 'time format'
+      def post_parse_text(tag)
+        results = @core.parse_text(options['format'], options['time_format'], options['keys'])
+        results.each do |result|
+          @logger.post(tag, result)
+        end
+      end
+
+      private
+
+      def parse_json(buffer)
+        begin
+          data = Yajl.load(buffer)
+        rescue => e
+          data = []
+          buffer.lines.each do |line|
+            data << Yajl.load(line)
+          end
+        end
+        data.class == Array ? data : [data]
+      end
+    end
+  end
+end

data/lib/fluent/client/core.rb

@@ -0,0 +1,47 @@
+module Fluent
+  module Client
+    class Core
+      def initialize
+      end
+
+      def create_parser(format, time_format, keys)
+        if format == 'multiline'
+          STDERR.puts 'multiline format is not support!'
+          exit 1
+        end
+
+        if format[0] == '/' && format[format.length - 1] == '/'
+          format = format.gsub(/^\/(.+)\/$/, '\1')
+          parser = Fluent::TextParser::RegexpParser.new(Regexp.new(format))
+        else
+          factory = Fluent::TextParser::TEMPLATE_REGISTRY.lookup(format)
+          parser = factory.call
+        end
+        parser.configure('time_format' => time_format) unless time_format.nil?
+        parser.configure('keys' => keys) unless keys.nil?
+        parser
+      end
+
+      def parse_text(format, time_format, keys, buffer = STDIN.read)
+        results = []
+        parser = create_parser(format, time_format, keys)
+        begin
+          buffer.lines.each_with_index do |line, i|
+            line = line.strip
+            parsed_time, parsed = parser.call(line)
+            if parsed.nil?
+              puts "format error! line #{i + 1} [#{line}]"
+            else
+              puts parsed
+            end
+            results << parsed unless parsed.nil?
+          end
+        rescue ArgumentError, RegexpError => e
+          STDERR.puts "\n#{e.message}\n#{e.backtrace.join("\n")}"
+          parsed_time = parsed = nil
+        end
+        results
+      end
+    end
+  end
+end

data/lib/fluent/client/version.rb

@@ -0,0 +1,6 @@
+module Fluent
+  module Client
+    # fluent-client version
+    VERSION = '0.1.0'
+  end
+end

data/spec/client_spec.rb

@@ -0,0 +1,16 @@
+require 'spec_helper'
+require 'fluent/client'
+
+describe Fluent::Client do
+  it 'should have a VERSION constant' do
+    subject.const_get('VERSION').should_not be_empty
+  end
+
+  it 'core parse_text csv' do
+    @core = Core.new
+    results = @core.parse_text('csv', nil, 'log_date,host,url', '2014-01-01,localhost,/hoge/test')
+    results.first['log_date'].should == '2014-01-01'
+    results.first['host'].should == 'localhost'
+    results.first['url'].should == '/hoge/test'
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,22 @@
+require 'rspec'
+require 'fluent/client/version'
+
+include Fluent::Client
+
+def capture_stdout
+  out = StringIO.new
+  $stdout = out
+  yield
+  return out.string
+ensure
+  $stdout = STDOUT
+end
+
+def capture_stderr
+  out = StringIO.new
+  $stderr = out
+  yield
+  return out.string
+ensure
+  $stderr = STDERR
+end

metadata

@@ -0,0 +1,203 @@
+--- !ruby/object:Gem::Specification
+name: fluent-client
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Hiroshi Toyama
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-09-14 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: thor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.19.1
+- !ruby/object:Gem::Dependency
+  name: fluentd
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: yajl-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: fluent-logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.7.2
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 10.3.2
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2.4'
+- !ruby/object:Gem::Dependency
+  name: rubygems-tasks
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.2'
+- !ruby/object:Gem::Dependency
+  name: yard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.8'
+description: fluentd command line utility
+email: toyama0919@gmail.com
+executables:
+- fluent-client
+extensions: []
+extra_rdoc_files: []
+files:
+- ".document"
+- ".gitignore"
+- ".rspec"
+- ".yardopts"
+- ChangeLog.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/fluent-client
+- fluent-client.gemspec
+- lib/fluent/client.rb
+- lib/fluent/client/cli.rb
+- lib/fluent/client/core.rb
+- lib/fluent/client/version.rb
+- spec/client_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/toyama0919/fluent-client
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: fluentd command line utility
+test_files:
+- spec/client_spec.rb
+- spec/spec_helper.rb