kaitai-struct-visualizer 0.5 → 0.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '0294e63ea9c355016ef56c48aed683eec1204ee4'
-  data.tar.gz: 49acc6230547d1c10c1978d626cb498a29d2cd0f
+  metadata.gz: 7b37b3d7fa10586b07b80076fb60a9b5bea5719a
+  data.tar.gz: 2ff461ac414e9ab3ecf4b0a712ad4e757acc27ff
 SHA512:
-  metadata.gz: 7962d68bdafde103d6cf56408436ec79299e3fc25ed7b76dee888bde59d06e65bc71231732d4c9fe6ba5dfd3ebb5c727c2f394aa9c081af1123c69f39fb20380
-  data.tar.gz: 07ac40819e35d59ae48e3fcdba3f965df4288ec0c6fe45867f236ec3859f0f06ed62b5e1ea5e6e59fdae80c4a1a7d688fd2ff7658da81767e61fb1b389e41302
+  metadata.gz: 2f9f6599576d9d6399cf5ceae258459cb8a26f038a1f95e7a2a1e4560e411b1b59e708026c75c6adb88efbf86cd38fcabeced8e6692868ca6c6495f725b37205
+  data.tar.gz: d276166a68b97fee40538360bea6717c70a22517972567f2a7886a629eb3b47d5c0bf04bf94ad6c36a5b849616f3e7f26a3aab818ead47fddf1472103d5a79ec

data/README.md

@@ -43,8 +43,7 @@ out source code in repository:
 
 ## Licensing
 
-Kaitai Struct visualizer itself is copyright (C) 2015-2016 Kaitai
-Project.
+Kaitai Struct visualizer is copyright (C) 2015-2017 Kaitai Project.
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU General Public License as published by

data/bin/ksdump

@@ -0,0 +1,126 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+
+# Some additional magic to make it work right after repo checkout,
+# without installation to proper Ruby library dirs
+
+full_bin_path = File.realpath($PROGRAM_NAME)
+dist_path = File.expand_path(File.dirname(full_bin_path) + '/..')
+
+$LOAD_PATH << "#{dist_path}/lib"
+$LOAD_PATH << "#{dist_path}/../runtime/ruby/lib"
+
+require 'kaitai/struct/visualizer'
+
+# ======================================================================
+
+options = {:format => 'yaml'}
+parser = OptionParser.new do |opts|
+  $prog_name = File.basename($PROGRAM_NAME)
+
+  opts.banner = "Usage: #{$prog_name} [options] <file_to_parse.bin> <format.ksy>..."
+  opts.separator ""
+
+  opts.on("-I", "--import-path [DIRECTORIES]", ".ksy library search path(s) for imports (see also KSPATH env variable)") do |v|
+    options[:import_path] = v
+  end
+
+  FORMATS = ['json', 'xml', 'yaml']
+  opts.on("-f", "--format FORMAT", FORMATS, "choose dump format - #{FORMATS.join(', ')} (default: #{options[:format]})") do |v|
+    options[:format] = v
+  end
+
+  opts.on("--version", "show versions of #{$prog_name} and ksc") do |v|
+    puts "#{$prog_name} #{Kaitai::Struct::Visualizer::VERSION}"
+    system("ksc", "--version")
+    exit 0
+  end
+end
+
+begin
+  parser.parse!
+rescue OptionParser::InvalidOption => e
+  puts e
+  puts parser
+  exit 1
+end
+
+if ARGV.size < 2
+  puts parser
+  exit 1
+end
+
+c = Kaitai::Struct::Visualizer::KSYCompiler.new(options)
+app = Kaitai::Struct::Visualizer::Parser.new(c, ARGV[0], ARGV[1..-1], options)
+app.load
+
+# ======================================================================
+
+def obj_to_h(obj)
+  if obj == true or obj == false or obj.is_a?(Numeric) or obj.nil?
+    obj
+  elsif obj.is_a?(Symbol)
+    obj.to_s
+  elsif obj.is_a?(String)
+    if obj.encoding == Encoding::ASCII_8BIT
+      r = ''
+      obj.each_byte { |x| r << sprintf('%02X ', x) }
+      r.chop!
+      r
+    else
+      obj.encode('UTF-8')
+    end
+  elsif obj.is_a?(Array)
+    obj.map { |x| obj_to_h(x) }
+  else
+    root = {}
+    valid_struct = false
+
+    common_meths = Set.new
+    obj.class.ancestors.each { |cl|
+      next if cl == obj.class
+      valid_struct = true if cl == Kaitai::Struct::Struct
+      common_meths.merge(cl.instance_methods)
+    }
+
+    return "OPAQUE (#{obj.class.to_s})" if not valid_struct
+
+    inst_meths = Set.new(obj.public_methods) - common_meths
+    inst_meths.sort.each { |meth|
+      k = meth.to_s
+      next if k =~ /^_/ # or attrs.include?(k)
+
+      el = obj.send(meth)
+      v = obj_to_h(el)
+      root[k] = v unless v.nil?
+    }
+
+#    obj.instance_variables.each { |k|
+#      k = k.to_s
+#      next if k =~ /^@_/
+#      el = obj.instance_eval(k)
+#      root[k[1..-1]] = obj_to_h(el)
+#    }
+    root
+  end
+end
+
+tree = obj_to_h(app.data)
+r = nil
+
+case options[:format]
+when 'json'
+  require 'json'
+  r = JSON.pretty_generate(tree)
+when 'xml'
+  require 'active_support'
+  require 'active_support/core_ext'
+  r = tree.to_xml
+when 'yaml'
+  require 'yaml'
+  r = tree.to_yaml
+  r = r[4..-1] if r[0..3] == "---\n"
+end
+
+print r

data/bin/ksv

@@ -1,5 +1,7 @@
 #!/usr/bin/env ruby
 
+require 'optparse'
+
 # Some additional magic to make it work right after repo checkout,
 # without installation to proper Ruby library dirs
 
@@ -11,10 +13,41 @@ $LOAD_PATH << "#{dist_path}/../runtime/ruby/lib"
 
 require 'kaitai/struct/visualizer'
 
+# ======================================================================
+
+options = {}
+parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{File.basename($PROGRAM_NAME)} [options] <file_to_parse.bin> <format.ksy>..."
+  opts.separator ""
+
+  opts.on("-I", "--import-path [DIRECTORIES]", ".ksy library search path(s) for imports (see also KSPATH env variable)") do |v|
+    options[:import_path] = v
+  end
+
+  opts.on("--opaque-types [BOOLEAN]", "opaque types allowed, default: false") do |v|
+    options[:opaque_types] = v
+  end
+
+  opts.on("--version", "show versions of ksv and ksc") do |v|
+    puts "kaitai-struct-visualizer #{Kaitai::Struct::Visualizer::VERSION}"
+    system("ksc", "--version")
+    exit 0
+  end
+end
+
+begin
+  parser.parse!
+rescue OptionParser::InvalidOption => e
+  puts e
+  puts parser
+  exit 1
+end
+
 if ARGV.size < 2
-  puts "Usage: #{File.basename($PROGRAM_NAME)} <file_to_parse.bin> <format.yaml>..."
+  puts parser
   exit 1
 end
 
-v = Kaitai::Struct::Visualizer::ExternalCompilerVisualizer.new(ARGV[0], ARGV[1..-1])
+c = Kaitai::Struct::Visualizer::KSYCompiler.new(options)
+v = Kaitai::Struct::Visualizer::Visualizer.new(c, ARGV[0], ARGV[1..-1], options)
 v.run

data/lib/kaitai/console_ansi.rb

@@ -9,15 +9,28 @@ class ConsoleANSI
   attr_reader :rows
 
   def initialize
-    # Normal POSIX way to determine console parameters
-    @cols = `tput cols`.to_i
-    @rows = `tput lines`.to_i
+    get_term_size
 
     @seq_clear = `tput clear`
     @seq_sgr0 = `tput sgr0`
 
     @seq_fgcolor = []
     @seq_bgcolor = []
+
+    @on_resize = nil
+
+    Signal.trap('SIGWINCH', proc {
+      get_term_size
+      @on_resize.call if @on_resize
+    })
+  end
+
+  def on_resize=(handler)
+    @on_resize = handler
+  end
+
+  def get_term_size
+    @rows, @cols = IO.console.winsize
   end
 
   def clear

data/lib/kaitai/console_windows.rb

@@ -50,6 +50,10 @@ class ConsoleWindows
     @rows = win_bottom - win_top + 1
   end
 
+  def on_resize=(handler)
+    @on_resize = handler
+  end
+
   def puts(s)
     Kernel::puts s
 #    num_written = 'XXXX'

data/lib/kaitai/struct/visualizer.rb

@@ -1,2 +1,3 @@
 require 'kaitai/struct/visualizer/version'
-require 'kaitai/struct/visualizer/visualizer_main'
+require 'kaitai/struct/visualizer/visualizer'
+require 'kaitai/struct/visualizer/ksy_compiler'

data/lib/kaitai/struct/visualizer/hex_viewer.rb

@@ -2,10 +2,12 @@ require 'kaitai/struct/visualizer/version'
 
 module Kaitai::Struct::Visualizer
 class HexViewer
-  def initialize(ui, buf, shift_x = 0, tree = nil)
+  attr_accessor :shift_x
+
+  def initialize(ui, buf, tree = nil)
     @ui = ui
     @buf = buf
-    @shift_x = shift_x
+    @shift_x = 0
     @tree = tree
 
     @embedded = not(tree.nil?)

data/lib/kaitai/struct/visualizer/ksy_compiler.rb

@@ -0,0 +1,116 @@
+require 'kaitai/struct/visualizer/version'
+require 'kaitai/tui'
+
+require 'open3'
+require 'json'
+require 'tmpdir'
+
+module Kaitai::Struct::Visualizer
+
+class KSYCompiler
+  def initialize(opts, out = $stderr)
+    @opts = opts
+    @out = out
+  end
+
+  def compile_formats(fns)
+    errs = false
+    main_class_name = nil
+    Dir.mktmpdir { |code_dir|
+      args = ['--ksc-json-output', '--debug', '-t', 'ruby', *fns, '-d', code_dir]
+
+      # Extra arguments
+      extra = []
+      extra += ['--import-path', @opts[:import_path]] if @opts[:import_path]
+      extra += ['--opaque-types', @opts[:opaque_types]] if @opts[:opaque_types]
+
+      args = extra + args
+
+      # UNIX-based systems run ksc via a shell wrapper that requires
+      # extra '--' in invocation to disambiguate our '-d' from java runner
+      # '-d' (which allows to pass defines to JVM). Windows-based systems
+      # do not need and do not support this extra '--', so we don't add it
+      # on Windows.
+      args.unshift('--') unless Kaitai::TUI::is_windows?
+
+      status = nil
+      log_str = nil
+      err_str = nil
+      Open3.popen3('kaitai-struct-compiler', *args) { |stdin, stdout, stderr, wait_thr|
+        status = wait_thr.value
+        log_str = stdout.read
+        err_str = stderr.read
+      }
+
+      if not status.success?
+        if status.exitstatus == 127
+          @out.puts "ksv: unable to find and execute kaitai-struct-compiler in your PATH"
+        elsif err_str =~ /Error: Unknown option --ksc-json-output/
+          @out.puts "ksv: your kaitai-struct-compiler is too old:"
+          system('kaitai-struct-compiler', '--version')
+          @out.puts "\nPlease use at least v0.7."
+        else
+          @out.puts "ksc crashed (exit status = #{status}):\n"
+          @out.puts "== STDOUT\n"
+          @out.puts log_str
+          @out.puts
+          @out.puts "== STDERR\n"
+          @out.puts err_str
+          @out.puts
+        end
+        exit status.exitstatus
+      end
+
+      log = JSON.load(log_str)
+
+      # FIXME: add log results check
+      @out.puts "Compilation OK"
+
+      fns.each_with_index { |fn, idx|
+        @out.puts "... processing #{fn} #{idx}"
+
+        log_fn = log[fn]
+        if log_fn['errors']
+          report_err(log_fn['errors'])
+          errs = true
+        else
+          log_classes = log_fn['output']['ruby']
+          log_classes.each_pair { |k, v|
+            if v['errors']
+              report_err(v['errors'])
+              errs = true
+            else
+              compiled_name = v['files'][0]['fileName']
+              compiled_path = "#{code_dir}/#{compiled_name}"
+
+              @out.puts "...... loading #{compiled_name}"
+              require compiled_path
+            end
+          }
+
+          # Is it main ClassSpecs?
+          if idx == 0
+            main = log_classes[log_fn['firstSpecName']]
+            main_class_name = main['topLevelName']
+          end
+        end
+      }
+
+    }
+
+    if errs
+      @out.puts "Fatal errors encountered, cannot continue"
+      exit 1
+    else
+      @out.puts "Classes loaded OK, main class = #{main_class_name}"
+    end
+
+    return main_class_name
+  end
+
+  def report_err(err)
+    @out.puts "Error: #{err.inspect}"
+  end
+end
+
+end

data/lib/kaitai/struct/visualizer/node.rb

@@ -186,6 +186,10 @@ class Node
         @pos2 = debug_el[:end]
       end
     elsif @value.is_a?(Array)
+      # Bail out early for empty array: it doesn't have proper
+      # debugging aids structure anyway
+      return if @value.empty?
+
       clean_id = @id[0] == '@' ? @id[1..-1] : @id
       debug_el = @parent.value._debug[clean_id]
       raise "Unable to get debugging aid for array: #{@parent.value._debug.inspect} using ID '#{clean_id}'" unless debug_el

data/lib/kaitai/struct/visualizer/parser.rb

@@ -0,0 +1,42 @@
+require 'kaitai/struct/visualizer/version'
+require 'kaitai/tui'
+require 'kaitai/struct/visualizer/tree'
+
+# TODO: should be inside compiled files
+require 'zlib'
+require 'stringio'
+
+module Kaitai::Struct::Visualizer
+
+##
+# Base class for everything that deals with compiling .ksy and parsing
+# stuff as object tree.
+class Parser
+  attr_reader :data
+
+  def initialize(compiler, bin_fn, formats_fn, opts)
+    @compiler = compiler
+    @bin_fn = bin_fn
+    @formats_fn = formats_fn
+    @opts = opts
+  end
+
+  def load
+    main_class_name = @compiler.compile_formats(@formats_fn)
+
+    main_class = Kernel::const_get(main_class_name)
+    @data = main_class.from_file(@bin_fn)
+
+    load_exc = nil
+    begin
+      @data._read
+    rescue EOFError => e
+      load_exc = e
+    rescue Kaitai::Struct::Stream::UnexpectedDataError => e
+      load_exc = e
+    end
+
+    return load_exc
+  end
+end
+end

data/lib/kaitai/struct/visualizer/tree.rb

@@ -11,17 +11,27 @@ class Tree
     @st = st
     @root = Node.new(self, st, 0)
     @root.id = '[root]'
-    @max_scr_ln = @ui.rows - 3
-
-    @hv_shift_x = @ui.cols - HexViewer.line_width - 1
 
     @cur_io = nil
-    @hv = HexViewer.new(ui, nil, @hv_shift_x, self)
+    @hv = HexViewer.new(ui, nil, self)
     @hv_hidden = false
 
+    recalc_sizes
+
     @cur_line = 0
     @cur_shift = 0
     @do_exit = false
+
+    @ui.on_resize = proc {
+      recalc_sizes
+      redraw
+      @hv.redraw
+    }
+  end
+
+  def recalc_sizes
+    @max_scr_ln = @ui.rows - 3
+    @hv.shift_x = @ui.cols - HexViewer.line_width - 1
   end
 
   def run
@@ -221,7 +231,7 @@ class Tree
     if @hv_hidden
       @ui.cols
     else
-      @hv_shift_x
+      @hv.shift_x
     end
   end
 

data/lib/kaitai/struct/visualizer/version.rb

@@ -1,7 +1,7 @@
 module Kaitai
   module Struct
     module Visualizer
-      VERSION = '0.5'
+      VERSION = '0.7'
     end
   end
 end

data/lib/kaitai/struct/visualizer/visualizer.rb

@@ -1,7 +1,6 @@
-require 'tmpdir'
-
 require 'kaitai/struct/visualizer/version'
 require 'kaitai/tui'
+require 'kaitai/struct/visualizer/parser'
 require 'kaitai/struct/visualizer/tree'
 
 # TODO: should be inside compiled files
@@ -9,38 +8,16 @@ require 'zlib'
 require 'stringio'
 
 module Kaitai::Struct::Visualizer
-class Visualizer
-  def initialize(bin_fn, formats_fn)
-    @bin_fn = bin_fn
-    @formats_fn = formats_fn
-    @primary_format = @formats_fn.shift
-
-    main_class_name = compile_format(@primary_format)
-
-    @formats_fn.each { |fn|
-      compile_format(fn)
-    }
-
-    main_class = Kernel::const_get(main_class_name)
-    @data = main_class.from_file(@bin_fn)
-
-    load_exc = nil
-    begin
-      @data._read
-    rescue EOFError => e
-      load_exc = e
-    rescue Kaitai::Struct::Stream::UnexpectedDataError => e
-      load_exc = e
-    end
+class Visualizer < Parser
+  def run
+    load_exc = load
 
     @ui = Kaitai::TUI.new
     @tree = Tree.new(@ui, @data)
 
     @tree.redraw
     @ui.message_box_exception(load_exc) if load_exc
-  end
 
-  def run
     @tree.run
   end
 end

data/lib/kaitai/tui.rb

@@ -33,6 +33,10 @@ class TUI
     end
   end
 
+  def on_resize=(handler)
+    @console.on_resize = handler
+  end
+
   def message_box_exception(e)
     message_box("Error while parsing", e.message)
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: kaitai-struct-visualizer
 version: !ruby/object:Gem::Version
-  version: '0.5'
+  version: '0.7'
 platform: ruby
 authors:
 - Mikhail Yakshin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2016-11-15 00:00:00.000000000 Z
+date: 2017-12-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -60,24 +60,26 @@ description: |
   This package is a visualizer tool for .ksy files. Given a particular binary file and .ksy file(s) that describe its format, it can visualize internal data structures in a tree form and a multi-level highlight hex viewer.
 email: greycat@kaitai.io
 executables:
+- ksdump
 - ksv
 extensions: []
 extra_rdoc_files: []
 files:
 - LICENSE
 - README.md
+- bin/ksdump
 - bin/ksv
 - kaitai-struct-visualizer.gemspec
 - lib/kaitai/console_ansi.rb
 - lib/kaitai/console_windows.rb
 - lib/kaitai/struct/visualizer.rb
 - lib/kaitai/struct/visualizer/hex_viewer.rb
+- lib/kaitai/struct/visualizer/ksy_compiler.rb
 - lib/kaitai/struct/visualizer/node.rb
+- lib/kaitai/struct/visualizer/parser.rb
 - lib/kaitai/struct/visualizer/tree.rb
 - lib/kaitai/struct/visualizer/version.rb
 - lib/kaitai/struct/visualizer/visualizer.rb
-- lib/kaitai/struct/visualizer/visualizer_main.rb
-- lib/kaitai/struct/visualizer/visualizer_ruby.rb
 - lib/kaitai/tui.rb
 homepage: http://kaitai.io
 licenses:
@@ -99,7 +101,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.1
+rubygems_version: 2.5.2
 signing_key: 
 specification_version: 4
 summary: Advanced hex viewer and binary structure exploration tool (visualizer) using

data/lib/kaitai/struct/visualizer/visualizer_main.rb

@@ -1,42 +0,0 @@
-require 'kaitai/struct/visualizer/version'
-require 'kaitai/struct/visualizer/visualizer'
-require 'kaitai/tui'
-
-module Kaitai::Struct::Visualizer
-
-class ExternalCompilerVisualizer < Visualizer
-  def compile_format(fn)
-    main_class_name = nil
-    Dir.mktmpdir { |code_dir|
-      args = ['--debug', '-t', 'ruby', fn, '-d', code_dir]
-
-      # UNIX-based systems run ksc via a shell wrapper that requires
-      # extra '--' in invocation to disambiguate our '-d' from java runner
-      # '-d' (which allows to pass defines to JVM). Windows-based systems
-      # do not need and do not support this extra '--', so we don't add it
-      # on Windows.
-      args.unshift('--') unless Kaitai::TUI::is_windows?
-
-      system('kaitai-struct-compiler', *args)
-      if $?.exitstatus != 0
-        st = $?.exitstatus
-        $stderr.puts("ksv: unable to find and execute kaitai-struct-compiler in your PATH") if st == 127
-        exit st
-      end
-
-      puts "Compilation OK"
-
-      compiled_path = Dir.glob("#{code_dir}/*.rb")[0]
-
-      require compiled_path
-
-      puts "Class loaded OK"
-
-      main_class_name = File.readlines(compiled_path).grep(/^class /)[0].strip.gsub(/^class /, '').gsub(/ <.*$/, '')
-    }
-
-    return main_class_name
-  end
-end
-
-end

data/lib/kaitai/struct/visualizer/visualizer_ruby.rb

@@ -1,18 +0,0 @@
-require 'ks_ruby_compiler'
-
-class RubyCompilerVisualizer extends Visualizer
-  def compile
-    Dir.mktmpdir { |code_dir|
-      compiled_path = "#{code_dir}/compiled.rb"
-      @compiler = CompileToRuby.new(@format_fn, compiled_path)
-      @compiler.compile
-
-      require compiled_path
-
-      main_class_name = @compiler.type2class(@compiler.desc['meta']['id'])
-      #puts "Main class: #{main_class_name}"
-      main_class = Kernel::const_get(main_class_name)
-      @data = main_class.from_file(@bin_fn)
-    }
-  end
-end