kaitai-struct-visualizer 0.3

data/README.md

@@ -0,0 +1,57 @@
+# Kaitai Struct: visualizer
+
+This is a simple visualizer for [Kaitai Struct](https://github.com/kaitai-io/kaitai_struct) project.
+
+Kaitai Struct is a declarative language used for describe various
+binary data structures, laid out in files or in memory: i.e. binary
+file formats, network stream packet formats, etc.
+
+The main idea is that a particular format is described in Kaitai
+Struct language (`.ksy` files) only once and then can be compiled with
+this compiler into source files in one of the supported programming
+languages. These modules will include a generated code for a parser
+that can read described data structure from a file / stream and give
+access to it in a nice, easy-to-comprehend API.
+
+Please refer to [documentation in Kaitai Struct project](https://github.com/kaitai-io/kaitai_struct)
+for details on `.ksy` files and general usage patterns.
+
+## Downloading and installing
+
+### Generic
+
+KS visualizer is written in Ruby and is available as .gem package. 
+
+### Source code
+
+If you're interested in developing the visualizer itself, you can check
+out source code in repository:
+
+    git clone https://github.com/kaitai-io/kaitai_struct_visualizer
+
+## Usage
+
+`ksv <binary-file> <ksy-file>...`
+
+## Licensing
+
+Kaitai Struct visualizer itself is copyright (C) 2015-2016 Kaitai
+Project.
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or (at
+your option) any later version.
+
+This program is distributed in the hope that it will be useful, but
+WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+Note that it applies only to compiler itself, not `.ksy` input files
+that one supplies in normal process of compilation, nor to compiler's
+output files — that consitutes normal usage process and you obviously
+keep copyright to both.

data/bin/ksv

@@ -0,0 +1,20 @@
+#!/usr/bin/env ruby
+
+# Some additional magic to make it work right after repo checkout,
+# without installation to proper Ruby library dirs
+
+full_bin_path = File.realpath($PROGRAM_NAME)
+dist_path = File.expand_path(File.dirname(full_bin_path) + '/..')
+
+$LOAD_PATH << "#{dist_path}/lib"
+$LOAD_PATH << "#{dist_path}/../runtime/ruby/lib"
+
+require 'kaitai/struct/visualizer'
+
+if ARGV.size < 2
+  puts "Usage: #{File.basename($PROGRAM_NAME)} <file_to_parse.bin> <format.yaml>..."
+  exit 1
+end
+
+v = Kaitai::Struct::Visualizer::ExternalCompilerVisualizer.new(ARGV[0], ARGV[1..-1])
+v.run

data/kaitai-struct-visualizer.gemspec

@@ -0,0 +1,37 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../lib/kaitai/struct/visualizer/version", __FILE__)
+require 'date'
+
+Gem::Specification.new { |s|
+  s.name = 'kaitai-struct-visualizer'
+  s.version = Kaitai::Struct::Visualizer::VERSION
+  s.date = Date.today.to_s
+
+  s.authors = ['Mikhail Yakshin']
+  s.email = 'greycat@kaitai.io'
+
+  s.homepage = 'http://kaitai.io'
+  s.summary = 'Advanced hex viewer and binary structure exploration tool (visualizer) using Kaitai Struct ksy files'
+  s.license = 'GPL-3.0+'
+  s.description = <<-EOF
+Kaitai Struct is a declarative language used for describe various binary data structures, laid out in files or in memory: i.e. binary file formats, network stream packet formats, etc.
+
+The main idea is that a particular format is described in Kaitai Struct language (.ksy file) and then can be compiled with ksc into source files in one of the supported programming languages. These modules will include a generated code for a parser that can read described data structure from a file / stream and give access to it in a nice, easy-to-comprehend API.
+
+This package is a visualizer tool for .ksy files. Given a particular binary file and .ksy file(s) that describe its format, it can visualize internal data structures in a tree form and a multi-level highlight hex viewer.
+EOF
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.require_paths = ['lib']
+
+  s.files = `git ls-files`.split("\n")
+  s.executables = s.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  s.add_development_dependency "bundler", "~> 1.3"
+  s.add_development_dependency 'rake', '~> 10'
+#  s.add_development_dependency 'rspec', '~> 3'
+
+  s.add_dependency 'kaitai-struct', "~> #{s.version}"
+}

data/lib/kaitai/struct/visualizer.rb

@@ -0,0 +1,2 @@
+require 'kaitai/struct/visualizer/version'
+require 'kaitai/struct/visualizer/visualizer_main'

data/lib/kaitai/struct/visualizer/hex_viewer.rb

@@ -0,0 +1,294 @@
+require 'kaitai/struct/visualizer/version'
+
+module Kaitai::Struct::Visualizer
+class HexViewer
+  def initialize(ui, buf, shift_x = 0, tree = nil)
+    @ui = ui
+    @buf = buf
+    @shift_x = shift_x
+    @tree = tree
+
+    @embedded = not(tree.nil?)
+    @max_scr_ln = @ui.rows - 3
+
+    @addr = 0
+    @scroll_y = 0
+    reset_cur
+    raise if @cur_x.nil?
+  end
+
+  def buf=(buf)
+    @buf = buf
+  end
+
+  def addr; @addr; end
+  def addr=(a)
+    @addr = a
+    reset_cur
+  end
+
+  def reset_cur
+    @cur_y = addr_to_row(@addr)
+    @cur_x = addr_to_col(@addr)
+  end
+
+  def highlight(regions)
+    highlight_hide
+    @hl_regions = regions
+    highlight_show
+  end
+
+  def ensure_visible
+    scr_y = row_to_scr(@cur_y)
+    if scr_y < 0
+      @scroll_y = @cur_y
+      redraw
+      highlight_show
+    elsif scr_y > @max_scr_ln
+      @scroll_y = @cur_y - @max_scr_ln
+      redraw
+      highlight_show
+    end
+  end
+
+  def run
+    c = nil
+    loop {
+      @ui.goto(0, @max_scr_ln + 1)
+      printf "%06x (%d, %d)", @addr, @cur_x, @cur_y
+
+      @ui.goto(col_to_col_char(@cur_x), row_to_scr(@cur_y))
+      c = @ui.read_char_mapped
+      case c
+      when :tab
+        return if @embedded
+      when :left_arrow
+        if @addr > 0
+          @addr -= 1
+          @cur_x -= 1
+          if @cur_x < 0
+            @cur_y -= 1
+            @cur_x = PER_LINE - 1
+          end
+        end
+      when :right_arrow
+        if @addr < @buf.size
+          @addr += 1
+          @cur_x += 1
+          if @cur_x >= PER_LINE
+            @cur_y += 1
+            @cur_x = 0
+          end
+        end
+      when :up_arrow
+        @addr -= PER_LINE
+        @cur_y -= 1
+        clamp_cursor
+      when :down_arrow
+        @addr += PER_LINE
+        @cur_y += 1
+        clamp_cursor
+      when :pg_dn
+        @addr += PER_LINE * PAGE_ROWS
+        @cur_y += PAGE_ROWS
+        clamp_cursor
+      when :pg_up
+        @addr -= PER_LINE * PAGE_ROWS
+        @cur_y -= PAGE_ROWS
+        clamp_cursor
+      when :home
+        if @cur_x == 0
+          @addr = 0
+          @cur_y = 0
+        else
+          @addr -= @cur_x
+          @cur_x = 0
+        end
+        clamp_cursor
+      when :end
+        if @cur_x == PER_LINE - 1
+          @addr = @buf.size - 1
+          reset_cur
+        else
+          @addr = @addr - @cur_x + PER_LINE - 1
+          @cur_x = PER_LINE - 1
+        end
+        clamp_cursor
+      when 'q'
+        @tree.do_exit if @tree
+        return
+      end
+
+      ensure_visible
+    }
+  end
+
+  def clamp_cursor
+    if @addr < 0
+      @addr = 0
+      @cur_x = 0
+      @cur_y = 0
+    elsif @addr >= @buf.size
+      @addr = @buf.size - 1
+      reset_cur
+    end
+  end
+
+  PER_LINE = 16
+  PER_GROUP = 4
+  PAGE_ROWS = 20
+  FMT = "%06x: %-#{PER_LINE * 3}s| %-#{PER_LINE}s\n"
+
+  def self.line_width
+    #6 + 2 + 3 * PER_LINE + 2 + PER_LINE
+    10 + 4 * PER_LINE
+  end
+
+  def col_to_col_hex(c)
+    #6 + 2 + 3 * c
+    @shift_x + 8 + 3 * c
+  end
+
+  def col_to_col_char(c)
+    #6 + 2 + 3 * PER_LINE + 2
+    @shift_x + 10 + 3 * PER_LINE + c
+  end
+
+  def row_to_scr(r)
+    r - @scroll_y
+  end
+
+  def redraw
+    i = row_col_to_addr(@scroll_y, 0)
+    row = 0
+
+    while row <= @max_scr_ln do
+      line = @buf[i, PER_LINE]
+      return unless line
+
+      @ui.goto(@shift_x, row)
+
+      hex = line.bytes.map { |x| sprintf('%02x', x) }.join(' ')
+      char = line.bytes.map { |x| byte_to_display_char(x) }.join
+
+      printf FMT, i, hex, char
+      i += PER_LINE
+      row += 1
+    end
+  end
+
+  def each_highlight_region
+    return if @hl_regions.nil?
+    n = @hl_regions.size
+    (n - 1).downto(0).each { |i|
+      p1 = @hl_regions[i][0]
+      p2 = @hl_regions[i][1]
+      yield i, p1, p2 unless p1.nil?
+    }
+  end
+
+  def highlight_hide
+    each_highlight_region { |i, p1, p2|
+      highlight_draw(p1, p2)
+    }
+  end
+
+  HIGHLIGHT_COLOR = [
+    :gray14,
+    :gray11,
+    :gray8,
+    :gray5,
+    :gray2,
+  ]
+
+  def highlight_show
+    each_highlight_region { |i, p1, p2|
+      @ui.bg_color = HIGHLIGHT_COLOR[i]
+      @ui.fg_color = :black
+      highlight_draw(p1, p2)
+    }
+    @ui.reset_colors
+  end
+
+  def highlight_draw(p1, p2)
+    r = row_to_scr(addr_to_row(p1))
+    return if r > @max_scr_ln
+    if r < 0
+      c = 0
+      r = 0
+      i = row_col_to_addr(@scroll_y, 0)
+      return if i >= p2
+    else
+      c = addr_to_col(p1)
+      i = p1
+    end
+
+    highlight_draw_hex(r, c, i, p2)
+    highlight_draw_char(r, c, i, p2)
+  end
+
+  def highlight_draw_hex(r, c, i, p2)
+    @ui.goto(col_to_col_hex(c), r)
+    while i < p2
+      v = byte_at(i)
+      return if v.nil?
+      printf('%02x ', v)
+      c += 1
+      if c >= PER_LINE
+        c = 0
+        r += 1
+        return if r > @max_scr_ln
+        @ui.goto(col_to_col_hex(c), r)
+      end
+      i += 1
+    end
+  end
+
+  def highlight_draw_char(r, c, i, p2)
+    @ui.goto(col_to_col_char(c), r)
+
+    while i < p2
+      v = byte_at(i)
+      return if v.nil?
+      print byte_to_display_char(v)
+      c += 1
+      if c >= PER_LINE
+        c = 0
+        r += 1
+        return if r > @max_scr_ln
+        @ui.goto(col_to_col_char(c), r)
+      end
+      i += 1
+    end
+  end
+
+  def byte_at(i)
+    v = @buf[i]
+    if v.nil?
+      nil
+    else
+      v.ord
+    end
+  end
+
+  def byte_to_display_char(x)
+    if x < 0x20 or x >= 0x7f
+      '.'
+    else
+      x.chr
+    end
+  end
+
+  def addr_to_row(addr)
+    addr / PER_LINE
+  end
+
+  def addr_to_col(addr)
+    addr % PER_LINE
+  end
+
+  def row_col_to_addr(row, col)
+    row * PER_LINE + col
+  end
+end
+end

data/lib/kaitai/struct/visualizer/node.rb

@@ -0,0 +1,242 @@
+# coding: utf-8
+require 'set'
+
+require 'kaitai/struct/visualizer/version'
+
+module Kaitai::Struct::Visualizer
+class Node
+  attr_accessor :id
+  attr_reader :value
+  attr_reader :level
+  attr_reader :pos1
+  attr_reader :pos2
+  attr_reader :children
+  attr_accessor :parent
+
+  def initialize(tree, value, level, value_method = nil, pos1 = nil, pos2 = nil)
+    @tree = tree
+    @value = value
+    @level = level
+    @value_method = value_method
+
+    unless pos1.nil? or pos2.nil?
+      @pos1 = pos1
+      @pos2 = pos2
+    end
+
+    @open = false
+    @explored = false
+
+    @children = []
+  end
+
+  def add(child)
+    @children << child
+    child.parent = self
+  end
+
+  def open?; @open; end
+
+  def openable?
+    not (
+      @value.is_a?(Fixnum) or
+      @value.is_a?(String) or
+      @value.is_a?(Symbol) or
+      @value === true or
+      @value === false
+    )
+  end
+
+  def hex?
+    @value.is_a?(String)
+  end
+
+  def toggle
+    if @open
+      close
+    else
+      open
+    end
+  end
+  
+  def open
+    return unless openable?
+    explore
+    @open = true if @explored
+  end
+
+  def close
+    @open = false
+  end
+
+  def draw(ui)
+    print '  ' * level
+    print(if @value.nil?
+          '[?]'
+         elsif open?
+          '[-]'
+         elsif openable?
+           '[+]'
+         else
+           '[.]'
+          end)
+    print " #{@id}"
+
+    pos = 2 * level + 4 + @id.length
+
+    if @value.is_a?(Fixnum)
+      print " = #{@value}"
+    elsif @value.is_a?(Symbol)
+      print " = #{@value}"
+    elsif @value.is_a?(String)
+      print ' = '
+      pos += 3
+      @str_mode = detect_str_mode unless @str_mode
+      max_len = @tree.tree_width - pos
+      case @str_mode
+      when :str
+        s = @value[0, max_len]
+      when :str_esc
+        s = @value.inspect[0, max_len]
+      when :hex
+        s = first_n_bytes_dump(@value, max_len / 3 + 1)
+      else
+        raise "Invalid str_mode: #{@str_mode.inspect}"
+      end
+      if s.length > max_len
+        s = s[0, max_len - 1]
+        s += '…'
+      end
+      print s
+    elsif @value === true or @value === false
+      print " = #{@value}"
+    elsif @value.is_a?(Array)
+      printf ' (%d = 0x%x entries)', @value.size, @value.size
+    end
+
+    puts
+  end
+
+  def first_n_bytes_dump(s, n)
+    i = 0
+    r = ''
+    s.each_byte { |x|
+      r << sprintf('%02x ', x)
+      i += 1
+      break if i >= n
+    }
+    r
+  end
+
+  ##
+  # Empirically detects a mode that would be best to show a designated string
+  def detect_str_mode
+    if @value.encoding == Encoding::ASCII_8BIT
+      :hex
+    else
+      :str_esc
+    end
+  end
+
+  def io
+    return @io if @io
+
+    if @parent.nil?
+      @io = @value._io
+    else
+      obj = @parent
+      while not obj.value.respond_to?(:_io)
+        obj = obj.parent
+      end
+      @io = obj.value._io
+    end
+  end
+
+  def explore
+    return if @explored
+
+    if @value.nil?
+      @value = @parent.value.send(@value_method)
+    end
+
+    if @value.is_a?(Fixnum) or @value.is_a?(String) or @value.is_a?(Symbol)
+      # do nothing else
+    elsif @value.is_a?(Array)
+      clean_id = @id[0] == '@' ? @id[1..-1] : @id
+      debug_el = @parent.value._debug[clean_id]
+      raise "Unable to get debugging aid for array: #{@parent.value._debug.inspect} using ID '#{clean_id}'" unless debug_el
+      aid = debug_el[:arr]
+      raise "Unable to get debugging aid for array: #{debug_el.inspect}" unless aid
+
+      max_val_digits = @value.size.to_s.size
+      fmt = "%#{max_val_digits}d"
+
+      @value.each_with_index { |el, i|
+        n = Node.new(@tree, el, level + 1, nil, aid[i][:start], aid[i][:end])
+        n.id = sprintf(fmt, i)
+        add(n)
+      }
+    else
+      # Gather seq attributes
+      attrs = Set.new
+      @value.instance_variables.each { |k|
+        k = k.to_s
+        next if k =~ /^@_/
+        el = @value.instance_eval(k)
+        aid = @value._debug[k[1..-1]]
+        if aid
+          aid_s = aid[:start]
+          aid_e = aid[:end]
+        else
+          #raise "Unable to get debugging aid for '#{k}'"
+          aid_s = nil
+          aid_e = nil
+        end
+        n = Node.new(@tree, el, level + 1, nil, aid_s, aid_e)
+        n.id = k
+        add(n)
+        attrs << k.gsub(/^@/, '')
+      }
+
+      # Gather instances
+      common_meths = Set.new
+      @value.class.ancestors.each { |cl|
+        next if cl == @value.class
+        common_meths.merge(cl.instance_methods)
+      }
+      inst_meths = Set.new(@value.public_methods) - common_meths
+      inst_meths.each { |meth|
+        k = meth.to_s
+        next if k =~ /^_/ or attrs.include?(k)
+        n = Node.new(@tree, nil, level + 1, meth)
+        n.id = k
+        add(n)
+      }
+    end
+    @explored = true
+  end
+
+  ##
+  # Determine total height of an element, including all children if
+  # it's open and visible
+  def height
+    if @open
+      r = 1
+      @children.each { |n| r += n.height }
+      r
+    else
+      1
+    end
+  end
+
+  ##
+  # Find out last (deepest) descendant of current node
+  def last_descendant
+    n = self
+    while n.open?
+      n = n.children.last
+    end
+    n
+  end
+end
+end