meslog 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 3ac08605cefe4c5f7b125cfb6cbe83a1caae6efd
+  data.tar.gz: 58ec3f7dd3cd86204ebd32ea45f566257b23cff1
+SHA512:
+  metadata.gz: 155a3e6a2c813e475d9b84a62ae18b712cf18a1b0aae958e13d21c5a09ec9aeb4a97d6221cdd604ee1c6ba409c6a35b286fd62163d44f3a8868225bcdc0de940
+  data.tar.gz: 38f564f6d4ba0f18194f36ce734041c1fc0679c41dd1bd75d989517940f78d8bbd92ca2b4f6fdceb232b9a3aeb909cefa2ea91ee0351433ab6dcd986baa3488a

data/.gitignore

@@ -0,0 +1,9 @@
+/.bundle/
+/.yardoc
+/Gemfile.lock
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,4 @@
+language: ruby
+rvm:
+  - 2.1.5
+before_install: gem install bundler -v 1.10.5

data/CODE_OF_CONDUCT.md

@@ -0,0 +1,13 @@
+# Contributor Code of Conduct
+
+As contributors and maintainers of this project, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities.
+
+We are committed to making participation in this project a harassment-free experience for everyone, regardless of level of experience, gender, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, or religion.
+
+Examples of unacceptable behavior by participants include the use of sexual language or imagery, derogatory comments or personal attacks, trolling, public or private harassment, insults, or other unprofessional conduct.
+
+Project maintainers have the right and responsibility to remove, edit, or reject comments, commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of Conduct. Project maintainers who do not follow the Code of Conduct may be removed from the project team.
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by opening an issue or contacting one or more of the project maintainers.
+
+This Code of Conduct is adapted from the [Contributor Covenant](http://contributor-covenant.org), version 1.0.0, available at [http://contributor-covenant.org/version/1/0/0/](http://contributor-covenant.org/version/1/0/0/)

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in meslog.gemspec
+gemspec

data/README.md

@@ -0,0 +1,61 @@
+# Meslog
+
+This project defines MESLOG data format, and provides the implementation of MESLOG data analyzer utility.
+
+## What is MESLOG
+
+MESLOG is designed for ease the pain of data summarization and analysis of experimental result.
+
+## MESLOG data format
+
+MESLOG data format is line-oriented text based format.
+Actually any text files are qualified to be MESLOG files.
+A MESLOG file is composed with (1) MESLOG data line and (2) normal line.
+An example of an MESLOG data line is like:
+
+    [MESLOG.sample]{"params":{"num_threads", 1},"data":{"flops": 1.0e9}}
+
+`[MESLOG.sample]` is a mark that means this line might be special data line tagged by "sample".
+After the mark, a string of JSON map object follows, which has two keys "params" and "data".
+Bit more formal definition of MESLOG data line and normal line is:
+  * Iff a string matches with `/\A\[MESLOG(\.[a-zA-Z0-9_-])?\]\Z/`, the string is "MESLOG header mark."
+  * Iff a JSON map object has only two key "params", "data", the object is "MESLOG compliant JSON object."
+  * Iff a line starts with a MESLOG header mark, followed by a string of a MESLOG compliant JSON object terminated by a newline character, the line is "MESLOG data line".
+  * If a line is not a MESLOG data line, the line is "normal line".
+
+
+
+Welcome to your new gem! In this directory, you'll find the files you need to be able to package up your Ruby library into a gem. Put your Ruby code in the file `lib/meslog`. To experiment with that code, run `bin/console` for an interactive prompt.
+
+TODO: Delete this and the text above, and describe your gem
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'meslog'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install meslog
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake rspec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/[USERNAME]/meslog. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](contributor-covenant.org) code of conduct.
+

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "meslog"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start

data/bin/setup

@@ -0,0 +1,7 @@
+#!/bin/bash
+set -euo pipefail
+IFS=$'\n\t'
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/exe/meslog

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require 'meslog'
+
+exit(Meslog::CLI::Runner.new.run(ARGV))

data/lib/meslog.rb

@@ -0,0 +1,7 @@
+require "meslog/version"
+require 'meslog/dataframe'
+require 'meslog/cli'
+
+module Meslog
+  # Your code goes here...
+end

data/lib/meslog/cli.rb

@@ -0,0 +1,362 @@
+# coding: utf-8
+
+require 'json'
+require 'optparse'
+
+module Meslog
+  module CLI
+    class Runner
+      def initialize
+      end
+
+      def run(argv)
+        $progname = $0
+
+        if argv.first == "-h" || argv.first == "--help"
+          puts "TODO: show help"
+        elsif argv.first == "agg"
+          $cmdname = argv.shift
+          return ::Meslog::Command::Agg.new.run(argv)
+        end
+      end
+    end
+  end
+
+  class DataCell
+    def initialize
+      @values = []
+    end
+
+    def push_value(val)
+      @values.push(val)
+    end
+
+    def avg
+      @values.inject(&:+) / @values.size.to_f
+    end
+
+    def size
+      @values.size
+    end
+  end
+
+  module Command
+    class BaseCommand
+      def initialize
+        @parser = OptionParser.new
+
+        # setup common options here
+        @x_axis = nil
+        @parser.on('-x', '--x-axis PARAM_PATH') do |param_path|
+          @x_axis = parse_param_path(param_path)
+        end
+
+        @select_matchers = []
+        @parser.on('--select MATCHER_EXP', 'Select only data lines which satisfies MATCHER_EXP and discard other lines. MATCHER_EXP ::= <param_path> \'=\' <param_val>') do |matcher_exp|
+          @select_matchers.push(parse_matcher_exp(matcher_exp))
+        end
+
+        @filter_matchers = []
+        @parser.on('--filter MATCHER_EXP') do |matcher_exp|
+          @filter_matchers.push(parse_matcher_exp(matcher_exp))
+        end
+      end
+
+      def parse_param_path(param_path_str)
+        param_path_str
+      end
+
+      def parse_matcher_exp(matcher_exp_str)
+        # TODO: refine after deciding specs of param_path
+
+        # bogus checking
+        unless matcher_exp_str =~ /^([a-z0-9_]+)=(.+)$/
+          raise ArgumentError.new("Invalid argument for MATCHER_EXP: #{matcher_exp_str}")
+        end
+
+        param_path = $~[1]
+        param_val_str = $~[2]
+
+        case param_val_str
+        when /^\d+$/
+          param_val = param_val_str.to_i
+        when /^[-+]?(?:[0-9]+(\.[0-9]*)?|(\.[0-9]+))([eE][-+]?[0-9]+)?$/
+          param_val = param_val_str.to_f
+        else
+          param_val = param_val_str
+        end
+
+        ret = {param_path: param_path, param_val: param_val}
+
+        def ret.match?(record)
+          record["params"][self[:param_path]] == self[:param_val]
+        end
+
+        ret
+      end
+
+      def fmt_num(x)
+        w = 6
+        w -= (Math.log10(x)).ceil
+        if w < 0
+          w = 0
+        end
+        fmtstr = "%.#{w}f"
+        sprintf(fmtstr, x)
+      end
+
+      def process_meslog(io_or_path)
+        if io_or_path.is_a? String
+          input = File.open(io_or_path)
+        else
+          input = io_or_path
+        end
+
+        records = []
+        param_paths = []
+        data_paths = []
+
+        # 1st pass: extract JSON records, parameter paths, and data paths
+        input.each_line do |line|
+          unless line =~ /^\[MESLOG(?:\.([a-z0-9_]+))?\](.+)$/
+            next
+          end
+
+          tag = $~[1]
+          json = JSON.parse($~[2])
+
+          if tag
+            raise NotImplementedError.new("tag is not implemented yet.")
+          end
+
+          record = json.dup
+          record["tag"] = tag
+
+          if @select_matchers.size > 0
+            matched = @select_matchers.any? do |matcher|
+              matcher.match?(record)
+            end
+
+            next unless matched
+          end
+
+          filtered = @filter_matchers.any? do |matcher|
+            matcher.match?(record)
+          end
+          next if filtered
+
+          records.push(record)
+
+          record["params"].keys.each do |param_path|
+            param_paths.push(param_path) unless param_paths.include?(param_path)
+          end
+
+          record["data"].keys.each do |data_path|
+            data_paths.push(data_path) unless data_paths.include?(data_path)
+          end
+        end
+
+        if records.size == 1
+          return true
+        end
+
+        const_param_paths = []
+        candidate_param_paths = []
+
+        fst_record = records.first
+        param_paths.each do |param_path|
+          if records.all? {|record|
+              fst_record["params"][param_path] == record["params"][param_path] }
+            const_param_paths.push(param_path)
+          else
+            candidate_param_paths.push(param_path)
+          end
+        end
+
+        # auto-selecting x-axis
+        if @x_axis.nil?
+          if candidate_param_paths.size == 0
+            puts "[ERROR] no candidate for x-axis."
+            exit(false)
+          elsif candidate_param_paths.size > 1
+            puts("[ERROR] multiple candidates for x-axis auto-selection. Specify one by --x-axis option.")
+            puts("  candidates: " + candidate_param_paths.join(", "))
+            exit(false)
+          end
+
+          @x_axis = candidate_param_paths.first
+        end
+
+        records = records.sort_by do |record|
+          record["params"][@x_axis]
+        end
+
+        # 2nd-pass: grouping records with parameter values excluding x-axis.
+        preset_param_paths = param_paths.select do |param_path|
+          param_path != @x_axis
+        end
+
+        record_groups = Hash.new
+        records.each do |record|
+          preset_params = preset_param_paths.map do |param_path|
+            record["params"][param_path]
+          end
+
+          record_groups[preset_params] ||= []
+          group = record_groups[preset_params]
+
+          group.push(record)
+        end
+
+        dataframe_list = []
+
+        # generate dataframe for each group
+        record_groups.each do |preset_params, group|
+          dataframe = ::Meslog::Dataframe.new(@x_axis, data_paths)
+
+          group.each do |record|
+            x = record["params"][@x_axis]
+
+            data_paths.each do |data_path|
+              dataframe[x][data_path] ||= DataCell.new
+              dataframe[x][data_path].push_value(record["data"][data_path])
+            end
+          end
+
+          preset_param_paths.each do |path|
+            idx = preset_param_paths.index(path)
+            val = preset_params[idx]
+            dataframe.preset_params[path] = val
+            if const_param_paths.include?(path)
+              dataframe.const_params[path] = val
+            end
+          end
+
+          dataframe_list.push(dataframe)
+        end
+
+        return dataframe_list
+      end
+    end
+
+    class Plot < BaseCommand
+      def initialize
+        super()
+
+        @parser.banner = <<EOS
+#{$progname} agg MESLOG_FILE [options]
+
+Options:
+EOS
+
+        @y_axis = nil
+        @parser.on('-y', '--y-axis PARAM_PATH') do |param_path|
+          @y_axis = parse_param_path(param_path)
+        end
+      end
+
+      def run(argv)
+        @parser.parse!(argv)
+
+        if argv.size == 0
+          print_help("MESLOG_FILE required.", true)
+        end
+
+        file = argv.shift
+
+        if file == "-"
+          file = $stdin
+        else
+          file = File.open(file)
+        end
+
+        ret = process_meslog(file)
+
+        record_groups = ret[:record_groups]
+        preset_param_paths = ret[:preset_param_paths]
+        const_param_paths = ret[:const_param_paths]
+        data_paths = ret[:data_paths]
+
+        record_groups.each do |preset_params, group|
+
+        end
+      end
+    end
+
+    class Agg < BaseCommand
+      def initialize
+        super()
+
+        @parser.banner = <<EOS
+#{$progname} agg MESLOG_FILE [options]
+
+Options:
+EOS
+      end
+
+      def plaintext_frame_label(dataframe)
+        const_param_label = dataframe.const_params.map do |key, val|
+          "#{key}: #{val}"
+        end.join(", ")
+
+        other_param_label = dataframe.preset_params.select do |key, val|
+          ! dataframe.const_params.has_key?(key)
+        end.map do |key, val|
+          "#{key} = #{val}"
+        end.join(", ")
+
+        if other_param_label.empty?
+          const_param_label
+        else
+          "#{const_param_label} || #{other_param_label}"
+        end
+      end
+
+      def run(argv)
+        @parser.parse!(argv)
+
+        if argv.size == 0
+          print_help("MESLOG_FILE required.", true)
+        end
+
+        file = argv.shift
+
+        if file == "-"
+          file = $stdin
+        else
+          file = File.open(file)
+        end
+
+        dataframe_list = process_meslog(file)
+
+        dataframe_list.each do |dataframe|
+          frame_label = plaintext_frame_label(dataframe)
+
+          puts("#== #{frame_label} ==")
+          puts(([dataframe.axis_path, "num_records"] + dataframe.data_paths).join("\t"))
+          dataframe.each do |x, data_cells|
+            puts([x,
+                 data_cells[dataframe.data_paths.first].size,
+                  *dataframe.data_paths.map{|path|
+                    fmt_num(data_cells[path].avg)
+                  }].map(&:to_s).join("\t"))
+          end
+
+          puts("")
+          puts("")
+        end
+
+        true
+      end
+
+      def print_help(errmsg = nil, do_exit = false)
+        if errmsg
+          $stderr.puts("[ERROR] #{errmsg}")
+        end
+        puts(@parser.help)
+
+        exit(false) if do_exit
+      end
+    end
+  end
+end

data/lib/meslog/dataframe.rb

@@ -0,0 +1,33 @@
+module Meslog
+  class Dataframe
+    attr_accessor :preset_params
+    attr_accessor :const_params
+    attr_reader :axis_path
+    attr_reader :data_paths
+
+    def initialize(axis_path, data_paths)
+      @records = Hash.new
+      @axis_path = axis_path
+      @data_paths = data_paths
+      @preset_params = Hash.new
+      @const_params = Hash.new
+    end
+
+    def [](key)
+      if @records.has_key?(key)
+        record = @records[key]
+      else
+        record = Hash.new
+        @records[key] = record
+      end
+
+      record
+    end
+
+    def each
+      @records.each do |key,val|
+        yield(key, val)
+      end
+    end
+  end
+end

data/lib/meslog/version.rb

@@ -0,0 +1,3 @@
+module Meslog
+  VERSION = "0.1.0"
+end

data/meslog.gemspec

@@ -0,0 +1,24 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'meslog/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "meslog"
+  spec.version       = Meslog::VERSION
+  spec.authors       = ["Yuto Hayamizu"]
+  spec.email         = ["y.hayamizu@gmail.com"]
+
+  spec.summary       = %q{Measurement result analyzer}
+  spec.description   = %q{MESLOG is an utility for analyzing measurement result data recorded in plain text log files.}
+  spec.homepage      = "https://github.com/hayamiz/meslog"
+
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_development_dependency "bundler", "~> 1.10"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,102 @@
+--- !ruby/object:Gem::Specification
+name: meslog
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yuto Hayamizu
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2016-10-19 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.10'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: MESLOG is an utility for analyzing measurement result data recorded in
+  plain text log files.
+email:
+- y.hayamizu@gmail.com
+executables:
+- meslog
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rspec"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- exe/meslog
+- lib/meslog.rb
+- lib/meslog/cli.rb
+- lib/meslog/dataframe.rb
+- lib/meslog/version.rb
+- meslog.gemspec
+homepage: https://github.com/hayamiz/meslog
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.1
+signing_key: 
+specification_version: 4
+summary: Measurement result analyzer
+test_files: []