buildlogparser 0.3.1 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: fb24a7d365529129348303b744a00d3eedeae902
-  data.tar.gz: 6fc6bbdd8e5742872ef4acd6d8e65c699c55f258
+SHA256:
+  metadata.gz: 75f73e2ee9abbf0130bfcbd90fd302a60e8033ca85155185ada98d1c058f915b
+  data.tar.gz: 3b6dce6aee391d67c076db96c27e81d282af9a5b6da68fcc5c1ae0f323781747
 SHA512:
-  metadata.gz: 6507122ba63a44041a03e91a2cf233c2b2c6cdc94c57aafe223a7db2ddc2c9111c8d4987049d2ab919ee2b9c0e12ecd9913641da4699731c07a96a332d22ce89
-  data.tar.gz: 8505eb1fb3d4171a7db944a5e38d78ccc43611cd181897dc2aee129d43263171097b02007580e0f20260691134ef6b1b86867a3c52e13e53e219eec5c30d4094
+  metadata.gz: 4de5af23fbcfae9462befb767609d5866ea50d39eb0ddafc042ddd7ab85e2eabefaddba8e00871379094d077f27f655e6fda34c585f2fb37e7dc9cccad7dad30
+  data.tar.gz: 59cb9a7e5903b32ad0a46491747717e599e4742b805ab6049397a6487de338e6b8307fb21522c202e4294c68a301331f02f4ab85f03481f8e6e05b8ae1c45482

data/CHANGELOG.md

@@ -1,5 +1,13 @@
 # Changelog
 
+## v0.4.0 (08.02.2020)
+
+* Updated command line parser library to new name.
+* [ctest/log] Relaxed the command syntax to allow all possible characters.
+* Added support for parsing *scimark2* command line output.
+* Added support for parsing lmbench *lat_mem_rd* command line output.
+* Added support for parsing *gem5* generated `stats.txt` files.
+
 ## v0.3.1 (11.05.2018)
 
 * Fixed broken __FILE__ check in command line tool.

data/README.md

@@ -77,5 +77,8 @@ $ RUBYLIB=<repo-path>/lib <repo-path>/bin/buildlogparser <arguments>
 * cmake/ctest 3.5.1
 * coremark 1.0
 * dhrystone (C Version) 2.1 and 2.2
+* gem5 (development version)
 * lld 7.0 (development version)
+* lmbench lat_mem_rd 3.0-a9
+* scimark (C Version) 2.0
 * size (GNU Binutils 2.26.1)

data/bin/buildlogparser

@@ -26,13 +26,13 @@ require 'yaml'
 require 'rubygems'
 require 'buildlogparser'
 require 'terminal-table'
-require 'trollop'
+require 'optimist'
 
 module ApplicationLogic
   def self.parseComandLine(argv)
     parsers = BuildLogParser::getParserNames
     formats = [:yaml, :csv, :table]
-    p = Trollop::Parser.new do
+    p = Optimist::Parser.new do
       banner <<-EOS
 Extracts information from various log file formats.
 
@@ -48,7 +48,7 @@ where [options] are:
       opt :parsermodule, "Path to ruby file or gem name that implements additional parsers.", :type => String, :multi => true, :short => "-p"
     end
     opts = {}
-    Trollop::with_standard_exception_handling p do
+    Optimist::with_standard_exception_handling p do
       opts = p.parse argv
     end
 

data/buildlogparser.gemspec

@@ -20,12 +20,12 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
   spec.bindir        = "bin"
 
-  spec.add_development_dependency "bundler", "~> 1.15"
-  spec.add_development_dependency "codecov", "~> 0.1.10"
-  spec.add_development_dependency "minitest", "~> 5.8"
-  spec.add_development_dependency "rake", "~> 10.5"
-  spec.add_development_dependency "simplecov", "~> 0.14"
+  spec.add_development_dependency "bundler", ">= 1.15"
+  spec.add_development_dependency "codecov", ">= 0.1.10"
+  spec.add_development_dependency "minitest", ">= 5.8"
+  spec.add_development_dependency "rake", ">= 10.5"
+  spec.add_development_dependency "simplecov", ">= 0.14"
+  spec.add_runtime_dependency "optimist", "~> 3.0"
   spec.add_runtime_dependency "parslet", "~> 1.8"
-  spec.add_runtime_dependency "trollop", "~> 2.1"
   spec.add_runtime_dependency "terminal-table", "~> 1.8"
 end

data/lib/buildlogparser.rb

@@ -26,5 +26,8 @@ require_relative "buildlogparser/parsers/cmake"
 require_relative "buildlogparser/parsers/coremark"
 require_relative "buildlogparser/parsers/ctest"
 require_relative "buildlogparser/parsers/dhrystone"
+require_relative "buildlogparser/parsers/gem5"
 require_relative "buildlogparser/parsers/lld"
+require_relative "buildlogparser/parsers/lmbench"
+require_relative "buildlogparser/parsers/scimark2"
 require_relative "buildlogparser/parsers/size"

data/lib/buildlogparser/parsers/ctest.rb

@@ -76,9 +76,9 @@ module BuildLogParser
 
       rule(:endofoutput) { newline.maybe >> str('<end of output>') >> newline }
       rule(:output)      { ( endofoutput.absent? >> any ).repeat }
-      rule(:endtime) { str('end time: ') >> restofline.as(:endtime) }
+      rule(:endtime)     { str('end time: ') >> restofline.as(:endtime) }
 
-      rule(:quotedcommand) { str('"') >> path.as(:commandstring) >> str('"') }
+      rule(:quotedcommand) { str('"') >> match['^\"'].repeat(1).as(:commandstring) >> str('"') }
       rule(:commandlist) { quotedcommand >> (space >> quotedcommand).repeat }
 
       rule(:event) do

data/lib/buildlogparser/parsers/gem5.rb

@@ -0,0 +1,131 @@
+#
+# Copyright (C) 2018 Mario Werner <nioshd@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+require 'parslet'
+
+module BuildLogParser
+  module Gem5Stats
+    class Parser < Parslet::Parser
+      rule(:space)     { match['\s'] }
+      rule(:space?)    { space.repeat }
+
+      rule(:newline)   { str("\r").maybe >> str("\n") }
+      rule(:restofline){ ( newline.absent? >> any ).repeat }
+
+      rule(:integer)   { str("-").maybe >> match['0-9'].repeat(1) }
+      rule(:float)     { integer >> match['\.,'] >> integer | str("nan") | str("inf") }
+
+      rule(:key)       { match['[:alnum:]=\+\.\-_\:'].repeat(1) }
+      rule(:kvpair)    { key.as(:key) >> space? >> (float.as(:float) | integer.as(:int)) }
+
+      rule(:target) do
+        str('---------- Begin Simulation Statistics ----------') >> newline >>
+          (kvpair >> restofline >> newline).repeat.as(:numbers)
+      end
+
+      rule(:start) { (target | (restofline >> newline | any).as(:drop)).repeat.as(:array) }
+      root(:start)
+    end # class Parser
+
+    def self.valid_symbol?(str)
+      str.match(/^\w+$/) != nil
+    end
+
+    def self.valid_symbol_sequence?(str)
+      parts = str.split('.').reject { |f| f.empty? }
+      parts.each { |e| return false unless Gem5Stats::valid_symbol?(e) }
+      return true
+    end
+
+    def self.to_symbol_sequence(str)
+      parts = str.split('.').reject { |f| f.empty? }
+      return parts.map { |e| e.to_sym }
+    end
+
+    def self.get_key_sequence(str)
+      # split the key string into a sequence of symbols and strings
+      keys = []
+      fragments = str.split("::").reject { |f| f.empty? }
+      fragments.each { |f|
+        if Gem5Stats::valid_symbol_sequence?(f)
+          keys.concat(Gem5Stats::to_symbol_sequence(f))
+        else
+          keys.push(f)
+        end
+      }
+      return keys
+    end
+
+    def self.insert_into_hash(hash, keys, value)
+      ptr = hash
+      keys[0..-2].each { |k|
+        ptr[k] ||= {}
+        ptr = ptr[k]
+      }
+      ptr[keys[-1]] = value
+    end
+
+    class Transform < Parslet::Transform
+      rule(:drop => subtree(:tree)) { }
+      rule(:numbers => subtree(:numbers)) do
+        result = {}
+        numbers.each { |e|
+          # extract the value with the correct type
+          if e.key?(:float)
+            str = e[:float]
+            if str == "nan"
+              value = Float::NAN
+            elsif str == "inf"
+              value = Float::INFINITY
+            else
+              value = Float(e[:float])
+            end
+          else
+            value = Integer(e[:int])
+          end
+
+          keys = Gem5Stats::get_key_sequence(String(e[:key]))
+          Gem5Stats::insert_into_hash(result, keys, value)
+        }
+        result
+      end
+      rule(:array => subtree(:tree)) { tree.compact.flatten }
+    end # class Transform
+  end # module Gem5Stats
+
+  class Gem5Parser < BuildLogParser::Parser
+    attr_reader :data
+
+    def reset()
+      super()
+      @data = []
+    end
+
+    def parseStats(logtext)
+      reset()
+      @logtext = logtext
+      parser = Gem5Stats::Parser.new
+      tree = parser.parse(logtext)
+      @data = Gem5Stats::Transform.new.apply(tree)
+    end
+  end # class Gem5Parser
+
+  registerParser(:gem5Stats, Gem5Parser, :parseStats)
+end # module BuildLogParser

data/lib/buildlogparser/parsers/lmbench.rb

@@ -0,0 +1,75 @@
+#
+# Copyright (C) 2018 Mario Werner <nioshd@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+require 'parslet'
+
+module BuildLogParser
+  module LmbenchLatMemRd
+    class Parser < Parslet::Parser
+      rule(:space)     { match['\s'] }
+      rule(:space?)    { space.repeat }
+
+      rule(:newline)   { str("\r").maybe >> str("\n") }
+      rule(:restofline){ ( newline.absent? >> any ).repeat }
+
+      rule(:integer)   { match['0-9'].repeat(1) }
+      rule(:float)     { integer >> (match['\.,'] >> integer).maybe }
+
+      rule(:kvpair)    { float.as(:key) >> space? >> float.as(:value) }
+
+      rule(:target) do
+        str('"stride=') >> integer.as(:stride) >> newline >>
+          (kvpair >> newline.maybe).repeat.as(:numbers)
+      end
+
+      rule(:start) { (target | (restofline >> newline | any).as(:drop)).repeat.as(:array) }
+      root(:start)
+    end # class Parser
+
+    class Transform < Parslet::Transform
+      rule(:drop => subtree(:tree)) { }
+      rule(:stride => simple(:stride), :numbers => subtree(:numbers)) do
+        result = { :stride => Integer(stride) }
+        numbers.each { |e| result[Float(e[:key])] = Float(e[:value]) }
+        result
+      end
+      rule(:array => subtree(:tree)) { tree.compact.flatten }
+    end # class Transform
+  end # module LmbenchLatMemRd
+
+  class LmbenchParser < BuildLogParser::Parser
+    attr_reader :data
+
+    def reset()
+      super()
+      @data = []
+    end
+
+    def parseLatMemRd(logtext)
+      reset()
+      @logtext = logtext
+      parser = LmbenchLatMemRd::Parser.new
+      tree = parser.parse(logtext)
+      @data = LmbenchLatMemRd::Transform.new.apply(tree)
+    end
+  end # class LmbenchParser
+
+  registerParser(:lmbenchLatMemRd, LmbenchParser, :parseLatMemRd)
+end # module BuildLogParser

data/lib/buildlogparser/parsers/scimark2.rb

@@ -0,0 +1,86 @@
+#
+# Copyright (C) 2018 Mario Werner <nioshd@gmail.com>
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+#
+require 'parslet'
+
+module BuildLogParser
+  module Scimark2
+    class Parser < Parslet::Parser
+      rule(:space)     { match['\s'] }
+      rule(:space?)    { space.repeat }
+
+      rule(:newline)   { str("\r").maybe >> str("\n") }
+      rule(:restofline){ ( newline.absent? >> any ).repeat }
+
+      rule(:integer)   { match['0-9'].repeat(1) }
+      rule(:float)     { integer >> (match['\.,'] >> integer).maybe }
+
+      rule(:target) do
+        str('Using') >> space? >> float.as(:time_per_kernel) >> str(' seconds min time per kenel.') >> newline >>
+        str('Composite Score:') >> space? >> float.as(:composite) >> newline >>
+        str('FFT') >> space? >> str('Mflops:') >> space? >> float.as(:fft) >> space? >>
+            str('(N=') >> integer.as(:fft_n) >> str(')') >> newline >>
+        str('SOR') >> space? >> str('Mflops:') >> space? >> float.as(:sor) >> space? >>
+            str('(') >> integer.as(:sor_1) >> space? >> str('x') >> space? >> integer.as(:sor_2) >> str(')') >> newline >>
+        str('MonteCarlo:') >> space? >> str('Mflops:') >> space? >> float.as(:monte_carlo) >> newline >>
+        str('Sparse matmult') >> space? >> str('Mflops:') >> space? >> float.as(:sparse_matmult) >> space? >>
+            str('(N=') >> integer.as(:sparse_matmult_n) >> str(', nz=') >> integer.as(:sparse_matmult_nz) >> str(')') >> newline >>
+        str('LU') >> space? >> str('Mflops:') >> space? >> float.as(:lu) >> space? >>
+            str('(M=') >> integer.as(:lu_m) >> str(', N=') >> integer.as(:lu_n) >> str(')')
+      end
+
+      rule(:start) { (target | (restofline >> newline | any).as(:drop)).repeat.as(:array) }
+      root(:start)
+    end # class Parser
+
+    class Transform < Parslet::Transform
+      rule(:drop => subtree(:tree)) { }
+      rule(:time_per_kernel => simple(:time_per_kernel), :composite => simple(:composite), :fft => simple(:fft), :fft_n => simple(:fft_n),
+        :sor => simple(:sor), :sor_1 => simple(:sor_1), :sor_2 => simple(:sor_2), :monte_carlo => simple(:monte_carlo),
+        :sparse_matmult => simple(:sparse_matmult), :sparse_matmult_n => simple(:sparse_matmult_n), :sparse_matmult_nz => simple(:sparse_matmult_nz),
+        :lu => simple(:lu), :lu_m => simple(:lu_m), :lu_n => simple(:lu_n)) do
+          { :time_per_kernel => Float(time_per_kernel), :composite => Float(composite), :fft => Float(fft), :fft_n => Integer(fft_n),
+            :sor => Float(sor), :sor_1 => Integer(sor_1), :sor_2 => Integer(sor_2), :monte_carlo => Float(monte_carlo),
+            :sparse_matmult => Float(sparse_matmult), :sparse_matmult_n => Integer(sparse_matmult_n), :sparse_matmult_nz => Integer(sparse_matmult_nz),
+            :lu => Float(lu), :lu_m => Integer(lu_m), :lu_n => Integer(lu_n) }
+      end
+      rule(:array => subtree(:tree)) { tree.compact.flatten }
+    end # class Transform
+  end # module Scimark2
+
+  class Scimark2Parser < BuildLogParser::Parser
+    attr_reader :data
+
+    def reset()
+      super()
+      @data = []
+    end
+
+    def parse(logtext)
+      reset()
+      @logtext = logtext
+      parser = Scimark2::Parser.new
+      tree = parser.parse(logtext)
+      @data = Scimark2::Transform.new.apply(tree)
+    end
+  end # class Scimark2Parser
+
+  registerParser(:scimark2, Scimark2Parser, :parse)
+end # module BuildLogParser

data/lib/buildlogparser/version.rb

@@ -21,8 +21,8 @@
 module BuildLogParser
   module VERSION #:nodoc:
     MAJOR = 0
-    MINOR = 3
-    PATCH = 1
+    MINOR = 4
+    PATCH = 0
 
     STRING = [MAJOR, MINOR, PATCH].join('.')
   end

metadata

@@ -1,113 +1,113 @@
 --- !ruby/object:Gem::Specification
 name: buildlogparser
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.0
 platform: ruby
 authors:
 - Mario Werner
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2018-05-11 00:00:00.000000000 Z
+date: 2020-02-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.15'
 - !ruby/object:Gem::Dependency
   name: codecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.10
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: 0.1.10
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.8'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '5.8'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '10.5'
 - !ruby/object:Gem::Dependency
   name: simplecov
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.14'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0.14'
 - !ruby/object:Gem::Dependency
-  name: parslet
+  name: optimist
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.8'
+        version: '3.0'
 - !ruby/object:Gem::Dependency
-  name: trollop
+  name: parslet
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.1'
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: terminal-table
   requirement: !ruby/object:Gem::Requirement
@@ -145,7 +145,10 @@ files:
 - lib/buildlogparser/parsers/coremark.rb
 - lib/buildlogparser/parsers/ctest.rb
 - lib/buildlogparser/parsers/dhrystone.rb
+- lib/buildlogparser/parsers/gem5.rb
 - lib/buildlogparser/parsers/lld.rb
+- lib/buildlogparser/parsers/lmbench.rb
+- lib/buildlogparser/parsers/scimark2.rb
 - lib/buildlogparser/parsers/size.rb
 - lib/buildlogparser/registry.rb
 - lib/buildlogparser/version.rb
@@ -169,7 +172,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.1
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Collection of various parsers for the extraction of information from build