msgknife 0.0.1 → 0.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 12486e66b362622c270279e3f899c878aefacbc2
-  data.tar.gz: 0f7c2744d6339331415be1642d4b9e050d1902cb
+  metadata.gz: 4fd10df9695dc9f79dacf92ee420f6982a2036b4
+  data.tar.gz: 3996af2ae260d1bf6903b6bddee0446dde52a4b9
 SHA512:
-  metadata.gz: 46957373ce19f947101ffd7e15880616c0c96659350615779c99b45df87afc680114c0c7cc7af35e89d9b78bcb3e68ddcaeb0e83f59dddfae9e9bb9a3e86886d
-  data.tar.gz: 635c553d8ba0ebb8b1ac6d58cdef331535f0cdf67a83f3960bc5e36cb27a06472d6bdea5ba8e81d9e8aeb56bbacfdc6adb686ab2ea988c1fc68f3c9b85e19f84
+  metadata.gz: 4fa624d4c51d6660a8fc842f094ca428dacbf15a0461e9e501197d48eed07515f9e4b93d1f45f7028ab77428ac358a054c169b22b54590d3f88c491d910eb037
+  data.tar.gz: d9ec2ad336ca7213b56b4dae51cfc95bfcc0771296bc8a7959f10ad07a2620c6c1968871a08f4c48f8c269b6c93d3d6b6a42d842062c43d5f65d0318a592229c

data/bin/mdot

@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+require 'set'
+require 'digest/md5'
+
+# MsgPack Prety Print
+class MDot < Msgknife::Stream
+  class Node
+    attr_reader :key, :hkey, :fwd_link, :fwd_count, :back_link, :category
+    
+    def initialize(label)
+      label = '(nil)' if label.nil?
+      @key = label
+      @hkey = Digest::MD5.hexdigest(label)
+      @fwd_link = {}
+      @fwd_count = Hash.new{|h,k| h[k] = 0}
+      @back_link = {}
+    end
+
+    def set_cat(cat)
+      @category = cat
+    end
+    
+    def attach(tgt)
+      @fwd_link[tgt.key] = tgt
+      @fwd_count[tgt.key] += 1
+      tgt.attach_back(self)
+    end
+
+    def attach_back(tgt)
+      @back_link[tgt.key] = tgt
+    end
+
+    def set(memo=Set.new)
+      if memo.include?(@key)
+        return memo
+      else
+        memo.add(@key)
+        @fwd_link.each do |k,n|
+          memo = n.set(memo) unless memo.include?(k)
+        end
+        @back_link.each do |k,n|
+          memo = n.set(memo) unless memo.include?(k)
+        end
+      end
+      return memo
+    end
+  end
+  
+  def initialize
+    super
+    @node_map = Hash.new{|h,k| h[k] = Node.new(k)}
+    @optpsr.on('-M val', 'size of minimum graph size') {|v| @min_size = v.to_i}
+    @optpsr.on('-N', 'skip if value is nil') {|v| @skip_nil = v }
+    @optpsr.on('-d', 'output as dot format') {|v| @out_dot = v }
+  end
+  
+  def setup(argv)
+    @key1 = argv[0]
+    @key2 = argv[1]
+  end
+
+  def recv(obj, ts, tag)
+    if obj.key?(@key1) and obj.key?(@key2)
+      return if @skip_nil and (obj[@key1].nil? or obj[@key2].nil?)
+      n1 = @node_map[obj[@key1]]
+      n2 = @node_map[obj[@key2]]
+      n1.set_cat(0)
+      n2.set_cat(1)
+      n1.attach(n2)
+    end
+  end
+
+  def teardown
+    ignore_set = Set.new
+    unless @min_size.nil?
+      @node_map.each do |key, node|
+        next if ignore_set.include?(key)
+        m = node.set
+        m.each {|n| ignore_set.add(n)} if m.size < @min_size
+      end
+    end
+    
+    if @out_dot
+      puts "digraph mdot {"
+      @node_map.each do |key, node|
+        next if ignore_set.include?(node.key)
+        key = node.key.gsub('"', '\"')
+        case node.category
+        when 0; color = '#fbeac1'
+        when 1; color = '#ecac12'
+        end
+        puts "  \"#{node.hkey}\" [label=\"#{key}\" color=\"#{color}\" style=\"filled\"];"
+      end
+
+      @node_map.each do |key1, node1|
+        next if ignore_set.include?(key1)
+        node1.fwd_link.each do |key2, node2|
+          count = node1.fwd_count[key2]
+          puts "  \"#{node1.hkey}\" -> \"#{node2.hkey}\" [label=\"#{count}\" len=2.5];"
+        end
+      end
+      puts "}"
+      
+    else
+      @node_map.each do |key1, node1|
+        next if ignore_set.include?(key1)
+        node1.fwd_link.each do |key2, node2|
+          count = node1.fwd_count[key2]
+          write_stream({'src'=>key1, 'dst'=>key2, 'count'=>count})
+        end
+      end
+    end
+  end
+end
+
+ins = MDot.new()
+ins.run(ARGV, 0..1)

data/bin/medit

@@ -0,0 +1,15 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MPP < Msgknife::Stream
+  def setup(argv); end
+  def recv(obj, ts, tag)
+    write_stream(obj, ts, tag)
+  end
+end
+
+ins = MPP.new()
+ins.run(ARGV)

data/bin/mhead

@@ -0,0 +1,30 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+class MHead < Msgknife::Stream
+  def initialize
+    super
+    @num = 20
+    @json = false
+    @optpsr.on('-n VAL', 'number of message to show') {|v| @num = v.to_i }
+  end
+  
+  def setup(argv)
+    @count = 0
+  end
+  def recv(obj, ts, tag)
+    if @count < @num
+      write_stream(obj, ts, tag)
+    else
+      return false
+    end
+    @count += 1
+  end
+  def teardown
+  end
+end
+
+MHead.new.run(ARGV)
+

data/bin/mheat

@@ -6,6 +6,7 @@ require 'pp'
 require 'time'
 require 'msgpack'
 require 'json'
+require 'terminfo'
 
 # MsgPack Prety Print
 class MHeatMap < Msgknife::Stream
@@ -19,19 +20,20 @@ class MHeatMap < Msgknife::Stream
     @ts_key = 'ts'
     @optpsr.on('-t VAL', 'timestamp key name') { |v| @ts_key = v }
 
+    @ts_start = nil
+    @ts_end   = nil
+    @optpsr.on('-s VAL', 'timestamp of start') {|v| @ts_start = v.to_i }
+    @optpsr.on('-e VAL', 'timestamp of end')   {|v| @ts_end   = v.to_i }
+
     @count_key = nil
     @optpsr.on('-c VAL', 'key name to sum numbers') { |v| @count_key = v }
 
-    @width = 80 - 32
+    @width = TermInfo.screen_size[1] - 35
     @optpsr.on('-w VAL', 'width') { |v| @width = v.to_i }
 
     @relative = false
     @optpsr.on('-r', 'relative time') { |v| @relative = true }
 
-    @output_fmt = 'console'
-    @optpsr.on('-j', 'output as json') { |v| @output_fmt = 'json' }
-    @optpsr.on('-m', 'output as msgpack') { |v| @output_fmt = 'msgpack' }
-
     # Set internval parameters
     @key = nil
     @ts_min = nil
@@ -68,7 +70,17 @@ class MHeatMap < Msgknife::Stream
   end
 
   def output_console(obj)
-    STDOUT.write(sprintf("%30s | %s", "", obj['cols'][0]))    
+    ep = obj['cols'].size - 1
+    dt1_len = obj['cols'][0].size
+    dt2_len = obj['cols'][ep].size
+    sp_len  = @width - (dt1_len + dt2_len) - 2
+    if sp_len > 0
+      STDOUT.write(sprintf("%30s | %s %s %s|" , "", obj['cols'][0], " " * sp_len,
+                           obj['cols'][ep]))
+    else
+      STDOUT.write(sprintf("%30s | %s" , "", obj['cols'][0]))
+    end
+
     puts
     STDOUT.write(sprintf("%30s | %s|", "-" * 28, 
                          "-" * @width))
@@ -95,6 +107,10 @@ class MHeatMap < Msgknife::Stream
   end
   
   def teardown
+    @ts_max = @ts_end   unless @ts_end.nil?
+    @ts_min = @ts_start unless @ts_start.nil?
+
+    raise 'start timestamp should be less than end timestamp' if @ts_max < @ts_min
     dur_ts = @ts_max - @ts_min
     ts_unit = dur_ts / @width
 
@@ -117,11 +133,13 @@ class MHeatMap < Msgknife::Stream
       }}
     @rows.each do |k, ts_list|
       ts_list.each do |ts, val|
-        idx = ((ts - @ts_min) / ts_unit).to_i
-        idx = @width - 1 if idx >= @width
-        data_map[k]['row'][idx] += val
-        if data_map[k]['max'] < data_map[k]['row'][idx]
-          data_map[k]['max'] = data_map[k]['row'][idx]
+        if @ts_min <= ts and ts <= @ts_max
+          idx = ((ts - @ts_min) / ts_unit).to_i
+          idx = @width - 1 if idx >= @width
+          data_map[k]['row'][idx] += val
+          if data_map[k]['max'] < data_map[k]['row'][idx]
+            data_map[k]['max'] = data_map[k]['row'][idx]
+          end
         end
       end
     end

data/bin/msplit

@@ -0,0 +1,37 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'fileutils'
+
+# MsgPack Prety Print
+class MGrep < Msgknife::Stream
+  def initialize
+    super
+    @optpsr.on('-a', 'append mode') { |v| @append = v }
+    @optpsr.on('-o VAL', 'output directory') { |v| @outdir = v }
+  end
+  
+  def setup(argv)
+    @key = argv[0]
+    @fd_map = Hash.new
+    if !(@outdir.nil?) and !(File.exists?(@outdir))
+      FileUtils.mkdir_p @outdir
+    end
+  end
+
+  def recv(obj, ts, tag)
+    val = obj.has_key?(@key) ? obj[@key] : 'unknown'
+    unless @fd_map.has_key?(val)
+      mode = @append ? 'wb+' : 'wb'
+      fpath = (val.nil?) ? 'nil.msg' : val + '.msg'
+      puts fpath
+      fpath = File.join(@outdir, fpath) unless @outdir.nil?
+      @fd_map[val] = File.open(fpath, mode)
+    end
+
+    write_stream(obj, ts, tag, @fd_map[val])
+  end
+end
+
+ins = MGrep.new()
+ins.run(ARGV, 0..0)

data/bin/mtext

@@ -0,0 +1,19 @@
+#!/usr/bin/env ruby
+
+require 'msgknife'
+require 'pp'
+
+# MsgPack Prety Print
+class MPP < Msgknife::Stream
+  def setup(argv)
+    @key = argv[0]
+  end
+  def recv(obj, ts, tag)
+    if obj.key?(@key) and (!@ignore_nil or !(obj[@key].nil?))
+      puts obj[@key]
+    end
+  end
+end
+
+ins = MPP.new()
+ins.run(ARGV, 0..0)

data/lib/msgknife/version.rb

@@ -1,3 +1,3 @@
 module Msgknife
-  VERSION = "0.0.1"
+  VERSION = "0.0.2"
 end

data/lib/msgknife.rb

@@ -1,7 +1,11 @@
+# -*- coding: utf-8 -*-
+
 require "msgknife/version"
 require 'msgpack'
 require 'optparse'
 require 'time'
+require 'pp'
+require 'json'
 
 module Msgknife
   class Stream
@@ -10,11 +14,14 @@ module Msgknife
     def initialize
       @out_encode = false
       @optpsr = OptionParser.new
-      @optpsr.on('-m', 'output as msgpack encode') { |v| @out_encode = v }
       @optpsr.on('-F', 'input as fluentd format')  { |v| @in_fluentd = v }
       @optpsr.on('-T VAL', 'key of timestamp in message')  { |v| @ts_key = v }
       @optpsr.on('-N', 'ignore if value is nil') {|v| @ignore_nil = v }
       @argv = []
+      
+      @output_fmt = 'console'
+      @optpsr.on('-j', 'output as json') { |v| @output_fmt = 'json' }
+      @optpsr.on('-m', 'output as msgpack') { |v| @output_fmt = 'msgpack' }
     end
 
     def run(cmd_argv, range = nil)
@@ -88,21 +95,17 @@ module Msgknife
       end
     end
 
-    def write_stream(obj, ts=nil, tag=nil)
-      if @out_encode == false
-        if ts.nil? and tag.nil?
-          pp obj
-        else
-          pp [tag, ts, obj]
-        end
-      else
-        if ts.nil? and tag.nil?
-          STDOUT.write(obj.to_msgpack)
-        else
-          STDOUT.write([tag, ts, obj].to_msgpack)
+    def write_stream(obj, ts=nil, tag=nil, io=STDOUT)
+      msg =(ts.nil? and tag.nil?) ? obj : [tag, ts, obj]
+
+      begin
+        case @output_fmt
+        when 'console'; PP.pp(msg, io);
+        when 'json';    JSON.dump(msg, io);
+        when 'msgpack'; io.write(msg.to_msgpack);
         end
+      rescue Errno::EPIPE => e ;
       end
-      rescue Errno::EPIPE => e ;      
     end
 
 

data/msgknife.gemspec

@@ -21,4 +21,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "bundler", "~> 1.7"
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_dependency "msgpack", '~> 0.5'
+  spec.add_dependency "ruby-terminfo"
+  spec.add_dependency "diff-lcs"  
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: msgknife
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.0.2
 platform: ruby
 authors:
 - Masayoshi Mizutani
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-02-07 00:00:00.000000000 Z
+date: 2015-04-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -52,15 +52,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '0.5'
+- !ruby/object:Gem::Dependency
+  name: ruby-terminfo
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: diff-lcs
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: MsgKnife makes easy to view, check and analyze MessagePack format file
   and stream such as Fluentd data
 email:
 - mizutani@sfc.wide.ad.jp
 executables:
+- mdot
+- medit
 - mentropy
+- mhead
 - mheat
 - mkeys
 - mpp
+- msplit
+- mtext
 - mval
 extensions: []
 extra_rdoc_files: []
@@ -70,10 +103,15 @@ files:
 - LICENSE.txt
 - README.md
 - Rakefile
+- bin/mdot
+- bin/medit
 - bin/mentropy
+- bin/mhead
 - bin/mheat
 - bin/mkeys
 - bin/mpp
+- bin/msplit
+- bin/mtext
 - bin/mval
 - lib/msgknife.rb
 - lib/msgknife/version.rb