msgknife 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 12486e66b362622c270279e3f899c878aefacbc2
+  data.tar.gz: 0f7c2744d6339331415be1642d4b9e050d1902cb
+SHA512:
+  metadata.gz: 46957373ce19f947101ffd7e15880616c0c96659350615779c99b45df87afc680114c0c7cc7af35e89d9b78bcb3e68ddcaeb0e83f59dddfae9e9bb9a3e86886d
+  data.tar.gz: 635c553d8ba0ebb8b1ac6d58cdef331535f0cdf67a83f3960bc5e36cb27a06472d6bdea5ba8e81d9e8aeb56bbacfdc6adb686ab2ea988c1fc68f3c9b85e19f84

data/.gitignore

@@ -0,0 +1,14 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+*.bundle
+*.so
+*.o
+*.a
+mkmf.log

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in msgknife.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2015 Masayoshi Mizutani
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,64 @@
+# Msgknife
+
+Utilities of MessagePack format file and stream.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'msgknife'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install msgknife
+
+## Usage
+
+### mpp: Msgpack Prittey Print
+
+	$ hexdump -C data.msg
+    0000000 82 a1 61 01 a1 62 01 82 a1 61 03 a1 62 04 82 a1
+    0000010 61 01 a1 62 08                                 
+    0000015
+    $ mpp data.msg
+    {"a"=>1, "b"=>1}
+    {"a"=>3, "b"=>4}
+    {"a"=>1, "b"=>8}
+
+### mkeys: Msgpack KEYS
+
+Count and show the number of each key fields.
+
+    $ mkeys data.msg
+	{"a"=>3, "b"=>3}
+
+### mentropy: Msgpack ENTROPY
+
+    $ mentropy data.msg              
+    {"a"=>0.9182958340544896, "b"=>1.584962500721156}
+
+
+## Common Options
+
+### `-m` output with msgpack encoding
+
+    $ mkeys data.msg
+	{"a"=>3, "b"=>3}
+    $ mkeys -m data.msg | hexdump    	
+    0000000 82 a1 61 03 a1 62 03                           
+    0000007
+
+
+## Contributing
+
+1. Fork it ( https://github.com/[my-github-username]/msgknife/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create a new Pull Request

data/Rakefile

@@ -0,0 +1,2 @@
+require "bundler/gem_tasks"
+

data/bin/mentropy

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MEntropy < Msgknife::Stream
+  def initialize
+    super
+
+    @optpsr.on('-t', 'tag mode') {|v| @tag_mode = v }
+    @optpsr.on('-r', 'show as ratio of maximum') {|v| @ratio = v }
+  end
+  
+  def setup(argv)
+    @key_map = Hash.new {|h1,k1| h1[k1] = Hash.new{|h2,k2| h2[k2] = 0}}
+  end
+
+  def recv(obj, ts, tag)
+    if @tag_mode
+      @key_map['tag'][tag] += 1 unless tag.nil?
+    else
+      obj.each do |k,v|
+        if !(@ignore_nil) or !(v.nil?)
+          @key_map[k][v] += 1
+        end
+      end
+    end
+  end
+
+  def teardown
+    res = {}
+    maxc = 0
+    @key_map.each do |k, m|
+      sum = (m.inject(0){|s,v| s + v[1]}).to_f
+      sump = 0.0
+      m.each do |val, count|
+        prob = count.to_f / sum
+        sump += prob * Math.log2(prob)
+      end
+      res[k] = -sump
+      maxc = sum if maxc < sum
+    end
+
+    maxp = -(Math.log2(1.0/maxc))
+    if @ratio
+      keys = res.keys
+      keys.each { |k| res[k] = res[k] / maxp }
+    end        
+        
+    write_stream(res);
+  end
+end
+
+ins = MEntropy.new()
+ins.run(ARGV)

data/bin/mheat

@@ -0,0 +1,146 @@
+#!/usr/bin/env ruby
+# coding: utf-8
+
+require 'msgknife'
+require 'pp'
+require 'time'
+require 'msgpack'
+require 'json'
+
+# MsgPack Prety Print
+class MHeatMap < Msgknife::Stream
+  BLANK = ' '
+  BARS = ['▁', '▂', '▃', '▄', '▅', '▆', '▇', '█']
+
+  def initialize
+    super
+
+    # Set options
+    @ts_key = 'ts'
+    @optpsr.on('-t VAL', 'timestamp key name') { |v| @ts_key = v }
+
+    @count_key = nil
+    @optpsr.on('-c VAL', 'key name to sum numbers') { |v| @count_key = v }
+
+    @width = 80 - 32
+    @optpsr.on('-w VAL', 'width') { |v| @width = v.to_i }
+
+    @relative = false
+    @optpsr.on('-r', 'relative time') { |v| @relative = true }
+
+    @output_fmt = 'console'
+    @optpsr.on('-j', 'output as json') { |v| @output_fmt = 'json' }
+    @optpsr.on('-m', 'output as msgpack') { |v| @output_fmt = 'msgpack' }
+
+    # Set internval parameters
+    @key = nil
+    @ts_min = nil
+    @ts_max = nil
+    @rows = Hash.new {|h,k| h[k] = Array.new}
+  end
+
+  def setup(argv)
+    @key = argv[0]
+  end
+
+  def add_ts(key, ts, val)
+
+    if key.instance_of? Array
+      key.each { |v| add_ts(v, ts, val) }
+    elsif key.instance_of? Hash
+      key.each { |k,v| add_ts(v, ts, val) }
+    elsif key.instance_of? String
+      @rows[key] << [ts, val]
+    elsif key.instance_of? Fixnum
+      @rows[key.to_s] << [ts, val]
+    end
+  end
+
+  def recv(msg, ts, tag)
+    @ts_min = ts if @ts_min.nil? or ts < @ts_min
+    @ts_max = ts if @ts_max.nil? or @ts_max < ts
+
+    if @count_key.nil?
+      add_ts(msg[@key], ts, 1)
+    else
+      add_ts(msg[@key], ts, msg[@count_key]) if msg.has_key? @count_key
+    end      
+  end
+
+  def output_console(obj)
+    STDOUT.write(sprintf("%30s | %s", "", obj['cols'][0]))    
+    puts
+    STDOUT.write(sprintf("%30s | %s|", "-" * 28, 
+                         "-" * @width))
+    puts
+
+    obj['rows'].each do |k, row|
+      label = sprintf("%30s | ", k[0..28])
+      STDOUT.write(label)
+      
+      v_max = row['row'].max
+
+      row['row'].each do |v|
+        if v == 0
+          STDOUT.write(BLANK)
+        else
+          bi = (v / v_max) * BARS.size
+          bi = BARS.size - 1 if BARS.size <= bi
+          STDOUT.write(BARS[bi])
+        end
+      end
+      STDOUT.write('|')
+      puts
+    end
+  end
+  
+  def teardown
+    dur_ts = @ts_max - @ts_min
+    ts_unit = dur_ts / @width
+
+    cols = Array.new(@width)
+    (0..@width-1).each do |i|
+      if @relative
+        ts = ts_unit * i
+        cols[i] = sprintf("%02d:%02d:%02d", (ts / 3600).to_i,
+                           ((ts % 3600) / 60).to_i, (ts % 60))
+      else
+        dt = Time.at(@ts_min + ts_unit * i)
+        cols[i] = dt.to_s.split(' ')[0..1].join(' ')
+      end
+    end
+    
+
+    data_map = Hash.new {|h,k| h[k] = {
+        'row' => Array.new(@width, 0),
+        'max' => 0,
+      }}
+    @rows.each do |k, ts_list|
+      ts_list.each do |ts, val|
+        idx = ((ts - @ts_min) / ts_unit).to_i
+        idx = @width - 1 if idx >= @width
+        data_map[k]['row'][idx] += val
+        if data_map[k]['max'] < data_map[k]['row'][idx]
+          data_map[k]['max'] = data_map[k]['row'][idx]
+        end
+      end
+    end
+
+    obj = {
+      'cols' => cols,
+      'rows' => data_map,
+      'ts_max' => @ts_max,
+      'ts_min' => @ts_min,
+    }
+
+    # write_stream(obj)
+    case @output_fmt
+    when 'console'; output_console(obj)
+    when 'json'; STDOUT.write(obj.to_json)
+    when 'msgpack'; STDOUT.write(obj.to_msgpack)
+    end
+  end
+end
+
+ins = MHeatMap.new()
+ins.run(ARGV, 0..0)

data/bin/mkeys

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MKEYS < Msgknife::Stream
+  def initialize
+    super
+
+    @optpsr.on('-t', 'tag mode') {|v| @tag_mode = v }
+    @optpsr.on('-N', 'ignore if value is nil') {|v| @ignore_nil = v }
+  end
+  
+  def setup(argv)
+    @key_map = Hash.new {|h1,k1| h1[k1] = 0}
+  end
+
+  def recv(obj, ts, tag)
+    if @tag_mode
+      @key_map[tag] += 1 unless tag.nil?
+    else
+      obj.each do |k,v|
+        if !(@ignore_nil) or !(v.nil?)
+          @key_map[k] += 1
+        end
+      end
+    end
+  end
+
+  def teardown
+    write_stream(@key_map);
+  end
+end
+
+ins = MKEYS.new()
+ins.run(ARGV)

data/bin/mpp

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MPP < Msgknife::Stream
+  def setup(argv); end
+  def recv(obj, ts, tag)
+    write_stream(obj, ts, tag)
+  end
+end
+
+ins = MPP.new()
+ins.run(ARGV)

data/bin/mval

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MPP < Msgknife::Stream
+  def setup(argv)
+    @key = argv[0]
+  end
+  def recv(obj, ts, tag)
+    if obj.key?(@key) and (!@ignore_nil or !(obj[@key].nil?))
+      write_stream({@key => obj[@key]}, ts, tag)
+    end
+  end
+end
+
+ins = MPP.new()
+ins.run(ARGV, 0..0)

data/lib/msgknife/version.rb

@@ -0,0 +1,3 @@
+module Msgknife
+  VERSION = "0.0.1"
+end

data/lib/msgknife.rb

@@ -0,0 +1,120 @@
+require "msgknife/version"
+require 'msgpack'
+require 'optparse'
+require 'time'
+
+module Msgknife
+  class Stream
+    attr_accessor :optpsr
+
+    def initialize
+      @out_encode = false
+      @optpsr = OptionParser.new
+      @optpsr.on('-m', 'output as msgpack encode') { |v| @out_encode = v }
+      @optpsr.on('-F', 'input as fluentd format')  { |v| @in_fluentd = v }
+      @optpsr.on('-T VAL', 'key of timestamp in message')  { |v| @ts_key = v }
+      @optpsr.on('-N', 'ignore if value is nil') {|v| @ignore_nil = v }
+      @argv = []
+    end
+
+    def run(cmd_argv, range = nil)
+      args = @optpsr.parse(cmd_argv)
+
+      unless range.nil?
+        raise "Not enough arguments" if args.size < range.last + 1
+        @argv = args.slice!(range)
+      end
+
+      setup(@argv)
+      read_stream(args)
+      teardown
+    end
+
+
+    def read_io(io)
+      u = MessagePack::Unpacker.new(io)
+      begin
+        u.each {|obj|
+          if @in_fluentd
+            recv(obj[2], obj[1], obj[0])
+          else
+            if @ts_key.nil? or !(obj.key?(@ts_key))
+              recv(obj, nil, nil)
+            else
+              ts = nil
+              ts_val = obj[@ts_key]
+              case ts_val
+              when String
+                dt = Time.parse(ts_val) rescue nil
+                ts = dt.to_i
+              when Fixnum
+                ts = ts_val
+              when Float
+                ts = ts_val.to_i
+              end
+                  
+              recv(obj, ts, nil)
+            end
+          end
+        }
+      rescue EOFError
+        # ignore
+      rescue Interrupt
+        return
+      end
+    end
+
+    def read_stream(files=nil)
+      if files == nil or files.size == 0
+        read_io(STDIN)
+      else
+        f_list = Array.new
+        if files.instance_of? String
+          f_list << files
+        else
+          f_list += files
+        end
+
+        f_list.each do |fpath|
+          if File.directory?(fpath)
+            Find.find(fpath) do |file|
+              next if File.directory?(file)
+              read_io(File.open(file, 'r'))
+            end
+          else
+            read_io(File.open(fpath, 'r'))
+          end 
+        end
+      end
+    end
+
+    def write_stream(obj, ts=nil, tag=nil)
+      if @out_encode == false
+        if ts.nil? and tag.nil?
+          pp obj
+        else
+          pp [tag, ts, obj]
+        end
+      else
+        if ts.nil? and tag.nil?
+          STDOUT.write(obj.to_msgpack)
+        else
+          STDOUT.write([tag, ts, obj].to_msgpack)
+        end
+      end
+      rescue Errno::EPIPE => e ;      
+    end
+
+
+    def setup(argv)
+      raise 'setup(argv) must be implemented' if argv.size > 0
+    end
+
+    def recv(obj, ts, tag)
+      raise 'exec(obj) must be implemented'
+    end
+
+    def teardown; end
+  end
+
+end

data/msgknife.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'msgknife/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "msgknife"
+  spec.version       = Msgknife::VERSION
+  spec.authors       = ["Masayoshi Mizutani"]
+  spec.email         = ["mizutani@sfc.wide.ad.jp"]
+  spec.summary       = %q{MsgKnife: Utilities of MessagePack format file and stream}
+  spec.description   = %q{MsgKnife makes easy to view, check and analyze MessagePack format file and stream such as Fluentd data}
+  spec.homepage      = "https://github.com/m-mizutani/msgknife"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files -z`.split("\x0")
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.7"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_dependency "msgpack", '~> 0.5'
+end

metadata

@@ -0,0 +1,105 @@
+--- !ruby/object:Gem::Specification
+name: msgknife
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Masayoshi Mizutani
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-02-07 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.5'
+description: MsgKnife makes easy to view, check and analyze MessagePack format file
+  and stream such as Fluentd data
+email:
+- mizutani@sfc.wide.ad.jp
+executables:
+- mentropy
+- mheat
+- mkeys
+- mpp
+- mval
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/mentropy
+- bin/mheat
+- bin/mkeys
+- bin/mpp
+- bin/mval
+- lib/msgknife.rb
+- lib/msgknife/version.rb
+- msgknife.gemspec
+homepage: https://github.com/m-mizutani/msgknife
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.5
+signing_key: 
+specification_version: 4
+summary: 'MsgKnife: Utilities of MessagePack format file and stream'
+test_files: []