resilience 0.0.2 → 0.1.1

data/lib/resilience/tables.rb

@@ -2,5 +2,6 @@
 # ReFS Tables
 # Copyright (C) 2015 Red Hat Inc.
 
+require 'resilience/tables/boot'
 require 'resilience/tables/object'
 require 'resilience/tables/system'

data/lib/resilience/tables/boot.rb

@@ -0,0 +1,89 @@
+#!/usr/bin/ruby
+# ReFS Boot Table
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class BootTable
+    include OnImage
+
+    SECTOR_SIZE = 0x200
+
+    def refs_offsets
+      offsets = []
+      fs_off = fs_offsets
+      image.offset = 0
+      fs_off.each do |address|
+        image.seek address
+        sig = image.read(FS_SIGNATURE.size).unpack('C*')
+        offsets << address if sig == FS_SIGNATURE
+      end
+      offsets
+    end
+
+    def refs_offset
+      refs_offsets.first
+    end
+  end
+
+  class MBR < BootTable
+    SIG_ADDRESS         =  0x1FE
+    SIG                 = [0x55, 0xAA]
+    PARTITION_ADDRESSES = [0x1C6, 0x1D6, 0x1E6, 0x1F6]
+
+    def detected?
+      image.seek SIG_ADDRESS
+      sig = image.read(2)
+      sig == SIG
+    end
+
+    def fs_offsets
+      offsets = []
+      PARTITION_ADDRESSES.each do |address|
+        image.seek address
+        address = image.read(4).unpack('V').first
+        offsets << address * SECTOR_SIZE unless address == 0
+      end
+      offsets
+    end
+
+    def gpt_offsets
+      offsets = []
+      fs_offsets.each do |address|
+        image.seek address
+        sig = image.read(GPT::SIG.size).unpack('C*')
+        offsets << address if sig == GPT::SIG
+      end
+      offsets
+    end
+
+    def gpt_offset
+      gpt_offsets.first
+    end
+  end # class MBR
+
+  class GPT < BootTable
+    NUM_ADDRESS      = 0x50
+    PARTITIONS_START = 0x200 # from the current image offset, the start of gpt partition
+    PARTITION_SIZE   = 0x80
+    OFFSET_ADDRESS   = 0x20
+
+    SIG = [0x45, 0x46, 0x49, 0x20, 0x50, 0x41, 0x52, 0x54] # EFI PART
+
+    def num
+      image.seek NUM_ADDRESS
+      image.read(4).unpack('V').first
+    end
+
+    def fs_offsets
+      offsets = []
+      0.upto(num-1) do |i|
+        image.seek PARTITIONS_START + i * PARTITION_SIZE
+        partition = image.read(PARTITION_SIZE).unpack('C*')
+        offset    = partition[OFFSET_ADDRESS...OFFSET_ADDRESS+8].pack('C*').unpack('V').first
+        offsets << offset * SECTOR_SIZE
+        break if offset == 0
+      end
+      offsets
+    end
+  end # class GPT
+end # module Resilience

data/lib/resilience/tables/object.rb

@@ -18,11 +18,15 @@ module Resilience
       table
     end
 
-    def parse_pages
+    # Depends on SystemTable extraction
+    def object_page_id
       # in the images I've seen this has always been the first entry
       # in the system table, though always has virtual page id = 2
       # which we could look for if this turns out not to be the case
-      object_page_id      = image.system_table.pages.first
+      image.system_table.pages.first
+    end
+
+    def parse_pages
       object_page_address = object_page_id * PAGE_SIZE
 
       # read number of objects from index header

data/lib/resilience/tables/system.rb

@@ -18,8 +18,12 @@ module Resilience
       table
     end
 
+    def self.first_page_address
+      PAGES[:first] * PAGE_SIZE
+    end
+
     def parse_pages
-      image.seek(FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
+      image.seek(self.class.first_page_address + ADDRESSES[:system_table_page])
       system_table_page    = image.read(8).unpack('Q').first
       system_table_address = system_table_page * PAGE_SIZE 
 
@@ -39,4 +43,3 @@ module Resilience
     end
   end # class SystemTable
 end # module Resilience
-

data/lib/resilience/trees.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby
+# ReFS Trees
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/trees/object_tree'

data/lib/resilience/trees/object_tree.rb

@@ -0,0 +1,45 @@
+#!/usr/bin/ruby
+# ReFS Object Tree
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class ObjectTree
+    include OnImage
+
+    attr_accessor :map
+
+    def initialize
+      @map ||= {}
+    end
+
+    def self.parse
+      tree = new
+      tree.parse_entries
+      tree
+    end
+
+    # Depends on Image Pages extraction
+    def page
+      image.pages.newest_for PAGES[:object_table]
+    end
+
+    def parse_entries
+      page.attributes.each { |attr|
+        obj1 = obj1_from attr
+        obj2 = obj2_from attr
+        @map[obj1] ||= []
+        @map[obj1]  << obj2
+      }
+    end
+
+    private
+
+    def obj1_from(attr)
+      attr.bytes[ADDRESSES[:object_tree_start1]..ADDRESSES[:object_tree_end1]]
+    end
+
+    def obj2_from(attr)
+      attr.bytes[ADDRESSES[:object_tree_start2]..ADDRESSES[:object_tree_end2]]
+    end
+  end # class ObjectTree
+end # module Resilience

metadata

@@ -1,50 +1,90 @@
 --- !ruby/object:Gem::Specification
 name: resilience
 version: !ruby/object:Gem::Version
-  version: 0.0.2
+  version: 0.1.1
 platform: ruby
 authors:
 - Mo Morsi
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-05-23 00:00:00.000000000 Z
-dependencies: []
+date: 2015-07-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: colored
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Ruby ReFS utils
 email: mmorsi@redhat.com
 executables:
-- fs.rb
-- resilience.rb
+- axe.rb
+- rarser.rb
+- reach.rb
+- pex.rb
 - rex.rb
-- rels.rb
-- ref.rb
-- clum.py
-- cle.rb
+- rinfo.rb
+- fcomp.rb
 extensions: []
 extra_rdoc_files: []
 files:
 - README.md
-- bin/cle.rb
-- bin/clum.py
-- bin/fs.rb
-- bin/ref.rb
-- bin/rels.rb
-- bin/resilience.rb
+- bin/axe.rb
+- bin/fcomp.rb
+- bin/pex.rb
+- bin/rarser.rb
+- bin/reach.rb
 - bin/rex.rb
+- bin/rinfo.rb
 - lib/resilience.rb
 - lib/resilience/attribute.rb
+- lib/resilience/cli/all.rb
+- lib/resilience/cli/bin/axe.rb
+- lib/resilience/cli/bin/fcomp.rb
+- lib/resilience/cli/bin/pex.rb
+- lib/resilience/cli/bin/rarser.rb
+- lib/resilience/cli/bin/reach.rb
+- lib/resilience/cli/bin/rex.rb
+- lib/resilience/cli/bin/rinfo.rb
+- lib/resilience/cli/conf.rb
+- lib/resilience/cli/default.rb
+- lib/resilience/cli/disk.rb
+- lib/resilience/cli/file.rb
+- lib/resilience/cli/image.rb
+- lib/resilience/cli/metadata.rb
+- lib/resilience/cli/output.rb
+- lib/resilience/collections/dirs.rb
+- lib/resilience/collections/files.rb
+- lib/resilience/collections/pages.rb
+- lib/resilience/conf.rb
 - lib/resilience/constants.rb
+- lib/resilience/core_ext.rb
 - lib/resilience/dirs.rb
 - lib/resilience/dirs/root_dir.rb
 - lib/resilience/fs_dir.rb
 - lib/resilience/fs_dir/dir_base.rb
+- lib/resilience/fs_dir/dir_entry.rb
+- lib/resilience/fs_dir/file_entry.rb
 - lib/resilience/fs_dir/record.rb
 - lib/resilience/image.rb
 - lib/resilience/mixins.rb
 - lib/resilience/mixins/on_image.rb
+- lib/resilience/page.rb
 - lib/resilience/tables.rb
+- lib/resilience/tables/boot.rb
 - lib/resilience/tables/object.rb
 - lib/resilience/tables/system.rb
+- lib/resilience/trees.rb
+- lib/resilience/trees/object_tree.rb
 homepage: https://gist.github.com/movitto/866de4356f56a3b478ca
 licenses:
 - MIT

data/bin/cle.rb

@@ -1,24 +0,0 @@
-#!/usr/bin/ruby
-# ReFS 0x4000 Cluster Extractor
-# By mmorsi - 2014-07-14
-
-CLUSTERS = [0x1e, 0x20, 0x21, 0x22, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
-            0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
-
-DISK   = 'flat-disk.vmdk'
-OUTDIR = 'clusters'
-
-REFS_VOLUME_OFFSET = 0x2100000
-
-inf  = File.open(DISK, 'rb')
-CLUSTERS.each do |cluster|
-  outf = File.open("#{OUTDIR}/#{cluster.to_s(16)}", 'wb')
-
-  offset = REFS_VOLUME_OFFSET + cluster * 0x4000
-  inf.seek(offset)
-  contents = inf.read(0x4000)
-  outf.write contents
-  outf.close
-end
-
-inf.close

data/bin/clum.py

@@ -1,28 +0,0 @@
-# ReFS 0x4000 Cluster Usage Mapper
-# By Willi Ballenthin  2012-03-25
-# With modifications by mmorsi - 2014-07-14
-import sys, struct
-
-#REFS_VOLUME_OFFSET = 0x2010000
-REFS_VOLUME_OFFSET = 0x2100000
-
-def main(args):
-    with open(args[1], "rb") as f:
-        global REFS_VOLUME_OFFSET
-        offset = REFS_VOLUME_OFFSET
-        cluster = 0
-        while True:
-            f.seek(offset + cluster * 0x4000)
-            buf = f.read(4)
-            if not buf: break
-            magic = struct.unpack("<I", buf)[0]
-            if magic == cluster:
-                print "Metadata cluster %s (%s)" % \
-                  (hex(cluster), hex(offset + cluster * 0x4000))
-            elif magic != 0:
-                print "Non-null cluster %s (%s)" % \
-                  (hex(cluster), hex(offset + cluster * 0x4000))
-            cluster += 1
-
-if __name__ == '__main__':
-    main(sys.argv)

data/bin/fs.rb

@@ -1,21 +0,0 @@
-#!/usr/bin/ruby
-# ReFS file searcher
-# By mmorsi - 2014-09-03
-
-DISK     = 'flat-disk.vmdk'
-OFFSET   = 0x2100000
-
-SEQUENCE_LENGTH = 8
-SEQUENCE = 0xe010002800000038 # inverted due to endian ordering
-
-inf  = File.open(DISK, 'rb')
-inf.seek(OFFSET)
-
-while check = inf.read(SEQUENCE_LENGTH)
-  check = check.unpack('Q').first
-  if check == SEQUENCE
-    puts 'File at: 0x' + inf.pos.to_s(16) + ' cluster ' + ((inf.pos - OFFSET) / 0x4000).to_s(16)
-  end
-end
-
-inf.close

data/bin/ref.rb

@@ -1,49 +0,0 @@
-#!/usr/bin/ruby
-# ReFS reference analyzer
-# By mmorsi - 2014-09-03
-
-require 'graphviz'
-
-CLUSTERS = [0x1e, 0x2e, 0x37, 0x38,
-            0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
-
-DISK     = 'flat-disk.vmdk'
-OFFSET   = 0x2100000
-
-inf  = File.open(DISK, 'rb')
-
-refs = {}
-
-CLUSTERS.each do |cluster|
-  offset = OFFSET + cluster * 0x4000
-  inf.seek(offset + 2) # skip first 2 bytes containing cluster #
-  while inf.pos < offset + 0x4000
-    checkb = inf.read(2).unpack('S').first
-    CLUSTERS.each do |checkc|
-      if checkb == checkc
-        refs[cluster]         ||= {}
-        refs[cluster][checkc] ||=  0
-        refs[cluster][checkc]  +=  1
-      end
-    end
-  end
-end
-
-inf.close
-
-g = GraphViz.new( :G, :type => :digraph )
-
-refs.keys.each { |cluster|
-  g.add_nodes cluster.to_s(16)
-}
-
-refs.keys.each { |cluster|
-  puts "cluster #{cluster.to_s(16)} references: "
-  refs[cluster].keys.each { |ref|
-    puts ref.to_s(16) + " (#{refs[cluster][ref]})"
-    #g.add_edges(cluster.to_s(16), ref.to_s(16))
-    g.add_edges(ref.to_s(16), cluster.to_s(16))
-  }
-}
-
-g.output :png => 'fan-out.png'

data/bin/rels.rb

@@ -1,269 +0,0 @@
-#!/usr/bin/ruby
-# ReFS File Lister
-# Copyright (C) 2014 Red Hat Inc.
-
-require 'optparse'
-require 'colored'
-
-FIRST_PAGE_ID =  0x1e
-PAGE_SIZE     =  0x4000
-FIRST_PAGE_ADDRESS = FIRST_PAGE_ID * PAGE_SIZE
-
-ROOT_DIR_ID   = [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0]
-DIR_ENTRY     = 0x20030
-FILE_ENTRY    = 0x10030
-
-DIR_TREE      = 0x301
-DIR_LIST      = 0x200
-#DIR_BRANCH    = 0x000 ?
-
-ADDRESSES  = {
-  # page
-  :virtual_page_number => 0x18,
-  :first_attr          => 0x30,
-
-  # page 0x1e
-  :system_table_page   => 0xA0,
-
-  # system table
-  :system_pages        => 0x58,
-
-  # generic table
-  :num_objects         => 0x20, # referenced from start of first attr
-
-  :table_length        => 0x04  # referenced from start of table header
-}
-
-def read_attribute(image)
-  pos      = image.pos
-  attr_len = image.read(4).unpack('L').first
-  return nil if attr_len == 0
-  image.seek pos
-
-  image.read(attr_len)
-end
-
-def record_boundries(attribute)
-  header        = attribute.unpack('S*')
-  key_offset    = header[2]
-  key_length    = header[3]
-  value_offset  = header[5]
-  value_length  = header[6]
-  [key_offset, key_length, value_offset, value_length]
-end
-
-def record_flags(attribute)
-  attribute.unpack('S*')[4]
-end
-
-def record_key(attribute)
-  ko, kl, vo, vl = record_boundries(attribute)
-  attribute.unpack('C*')[ko...ko+kl].pack('C*')
-end
-
-def record_value(attribute)
-  ko, kl, vo, vl = record_boundries(attribute)
-  attribute.unpack('C*')[vo..-1].pack('C*')
-end
-
-def filter_dir_record(dir_entry, opts)
-  image = opts[:image]
-
-  # '4' seems to indicate a historical record or similar,
-  # records w/ flags '0' or '8' are what we want
-  record_flags(dir_entry)== 4 ? filter_dir_record(read_attribute(opts[:image]), opts) : dir_entry 
-end
-
-def parse_dir_branch(node, prefix, opts)
-  key          = record_key(node)
-  value        = record_value(node)
-  flags        = record_flags(node)
-
-  value_dwords = value.unpack('L*')
-  value_qwords = value.unpack('Q*')
-
-  page_id      = value_dwords[0]
-  page_address = page_id * PAGE_SIZE
-  checksum     = value_qwords[2]
-  parse_dir_page page_address, prefix, opts unless checksum == 0 || flags == 4
-end
-
-def parse_dir_record(dir_entry, prefix, opts)
-  key        = record_key(dir_entry)
-  value      = record_value(dir_entry)
-
-  key_bytes  = key.unpack('C*')
-  key_dwords = key.unpack('L*')
-  entry_type = key_dwords.first
-  if entry_type == DIR_ENTRY
-    dir_name = key_bytes[4..-1].pack('L*')
-    opts[:dirs] << "#{prefix}\\#{dir_name}"
-
-    dir_obj = value.unpack('C*')[0...8]
-    dir_obj = [0, 0, 0, 0, 0, 0, 0, 0].concat(dir_obj)
-    parse_dir_obj(dir_obj, "#{prefix}\\#{dir_name}", opts)
-
-  elsif entry_type == FILE_ENTRY
-    file_name = key_bytes[4..-1].pack('L*')
-    opts[:files] << "#{prefix}\\#{file_name}"
-  end
-end
-
-def parse_dir_obj(object_id, prefix, opts)
-  image        = opts[:image]
-  object_pages = opts[:object_pages]
-  opts[:dirs]  ||= []
-  opts[:files] ||= []
-
-  page_id = object_pages[object_id]
-  page_address = page_id * PAGE_SIZE
-  parse_dir_page page_address, prefix, opts
-end
-
-def parse_dir_page(page_address, prefix, opts)
-  image        = opts[:image]
-  image_start  = opts[:offset]
-
-  # skip container/placeholder attribute
-  image.seek(image_start + page_address + ADDRESSES[:first_attr])
-  read_attribute(image)
-
-  # start of table attr, pull out table length, type
-  table_header_attr   = read_attribute(image)
-  table_header_dwords = table_header_attr.unpack("L*")
-  header_len          = table_header_dwords[0]
-  table_len           = table_header_dwords[1]
-  remaining_len       = table_len - header_len
-  table_type          = table_header_dwords[3]
-
-  until remaining_len == 0
-    orig_pos = image.pos
-    record   = read_attribute(image)
-
-    # need to keep track of position locally as we
-    # recursively call parse_dir via helpers
-    pos = image.pos
-
-    if table_type == DIR_TREE
-      parse_dir_branch record, prefix, opts
-    else #if table_type == DIR_LIST
-      record = filter_dir_record(record, opts)
-      pos = image.pos
-      parse_dir_record record, prefix, opts
-
-    end
-
-    image.seek pos
-    remaining_len -= (image.pos - orig_pos)
-  end
-end
-
-def parse_object_table(opts)
-  image       = opts[:image]
-  image_start = opts[:offset]
-  opts[:object_pages] ||= {}
-
-  # in the images I've seen this has always been the first entry
-  # in the system table, though always has virtual page id = 2
-  # which we could look for if this turns out not to be the case
-  object_page_id = opts[:system_pages].first
-  object_page_address = object_page_id * PAGE_SIZE
-
-  # read number of objects from index header
-  image.seek(image_start + object_page_address + ADDRESSES[:first_attr])
-  first_attr  = read_attribute(image)
-  num_objects = first_attr.unpack('L*')[ADDRESSES[:num_objects]/4]
-
-  # start of table attr, skip for now
-  read_attribute(image)
-
-  0.upto(num_objects-1) do
-    object_record  = read_attribute(image)
-    object_id      = record_key(object_record).unpack('C*')
-
-    # here object page is first qword of record value
-    object_page    = record_value(object_record).unpack('Q*').first
-    opts[:object_pages][object_id] = object_page
-  end
-end
-
-def parse_system_table(opts)
-  image       = opts[:image]
-  image_start = opts[:offset]
-  opts[:system_pages] ||= []
-
-  image.seek(image_start + FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
-  system_table_page = image.read(8).unpack('Q').first
-  system_table_address = system_table_page * PAGE_SIZE 
-
-  image.seek(image_start + system_table_address + ADDRESSES[:system_pages])
-  num_system_pages = image.read(4).unpack('L').first
-
-  0.upto(num_system_pages-1) do
-    system_page_offset = image.read(4).unpack('L').first
-    pos = image.pos
-
-    image.seek(image_start + system_table_address + system_page_offset)
-    system_page = image.read(8).unpack('Q').first
-    opts[:system_pages] << system_page
-
-    image.seek(pos)
-  end
-end
-
-def print_results(opts)
-  puts "Dirs found:".bold
-  opts[:dirs].each { |dir|
-    puts "#{dir}".blue
-  }
-
-  puts
-  puts "Files found:".bold
-  opts[:files].sort.each { |file|
-    puts "#{file}".red
-  }
-end
-
-def main(opts = {})
-  image = File.open(opts[:image], 'rb')
-  opts[:image] = image
-
-  parse_system_table(opts)
-  parse_object_table(opts)
-  parse_dir_obj(ROOT_DIR_ID, '', opts)
-  print_results(opts)
-end
-
-def parse_cli(cli)
-  opts   = {}
-  parser = OptionParser.new do |popts|
-    popts.on("-h", "--help", "Print help message") do
-      puts parser
-      exit
-    end
-
-    popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
-      opts[:image] = path
-    end
-
-    popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
-      opts[:offset] = offset.to_i
-    end
-  end
-
-  begin
-    parser.parse!(cli)
-  rescue OptionParser::InvalidOption
-    puts parser
-    exit
-  end
-
-  if !opts[:image] || !opts[:offset]
-    puts "--image and --offset params are needed at a minimum"
-    exit 1
-  end
-
-  opts
-end
-
-main parse_cli(ARGV) if __FILE__ == $0