resilience 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 4ce546fb9a6dcfd401ccb8adbfa16bb3147f3a3b
+  data.tar.gz: 6c47f4c2c0c3f89d74a30335928ef8bb991dcd0b
+SHA512:
+  metadata.gz: 7d25ff402e660223d44333beab243b7281d8382a8a2f31caeea6670e1adef46fc6fd54d218475ecd2b0f7061148b2f0111b297e45ffcbc412886920dbcf23bee
+  data.tar.gz: 9767ac37ea16d858c02d89a217d33a8959509f03c1e7974d8b37d81db05b50576b8501a69ea1392338afafff7a06fae8c8a58dee3418198af1625d5120a1d199

data/README.md

@@ -0,0 +1,3 @@
+Resilience - Experimental ReFS Library  
+Copyright (C) 2014-2015 Red Hat Inc.  
+Licensed under the MIT license

data/bin/cle.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/ruby
+# ReFS 0x4000 Cluster Extractor
+# By mmorsi - 2014-07-14
+
+CLUSTERS = [0x1e, 0x20, 0x21, 0x22, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
+            0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
+
+DISK   = 'flat-disk.vmdk'
+OUTDIR = 'clusters'
+
+REFS_VOLUME_OFFSET = 0x2100000
+
+inf  = File.open(DISK, 'rb')
+CLUSTERS.each do |cluster|
+  outf = File.open("#{OUTDIR}/#{cluster.to_s(16)}", 'wb')
+
+  offset = REFS_VOLUME_OFFSET + cluster * 0x4000
+  inf.seek(offset)
+  contents = inf.read(0x4000)
+  outf.write contents
+  outf.close
+end
+
+inf.close

data/bin/clum.py

@@ -0,0 +1,28 @@
+# ReFS 0x4000 Cluster Usage Mapper
+# By Willi Ballenthin  2012-03-25
+# With modifications by mmorsi - 2014-07-14
+import sys, struct
+
+#REFS_VOLUME_OFFSET = 0x2010000
+REFS_VOLUME_OFFSET = 0x2100000
+
+def main(args):
+    with open(args[1], "rb") as f:
+        global REFS_VOLUME_OFFSET
+        offset = REFS_VOLUME_OFFSET
+        cluster = 0
+        while True:
+            f.seek(offset + cluster * 0x4000)
+            buf = f.read(4)
+            if not buf: break
+            magic = struct.unpack("<I", buf)[0]
+            if magic == cluster:
+                print "Metadata cluster %s (%s)" % \
+                  (hex(cluster), hex(offset + cluster * 0x4000))
+            elif magic != 0:
+                print "Non-null cluster %s (%s)" % \
+                  (hex(cluster), hex(offset + cluster * 0x4000))
+            cluster += 1
+
+if __name__ == '__main__':
+    main(sys.argv)

data/bin/fs.rb

@@ -0,0 +1,21 @@
+#!/usr/bin/ruby
+# ReFS file searcher
+# By mmorsi - 2014-09-03
+
+DISK     = 'flat-disk.vmdk'
+OFFSET   = 0x2100000
+
+SEQUENCE_LENGTH = 8
+SEQUENCE = 0xe010002800000038 # inverted due to endian ordering
+
+inf  = File.open(DISK, 'rb')
+inf.seek(OFFSET)
+
+while check = inf.read(SEQUENCE_LENGTH)
+  check = check.unpack('Q').first
+  if check == SEQUENCE
+    puts 'File at: 0x' + inf.pos.to_s(16) + ' cluster ' + ((inf.pos - OFFSET) / 0x4000).to_s(16)
+  end
+end
+
+inf.close

data/bin/ref.rb

@@ -0,0 +1,49 @@
+#!/usr/bin/ruby
+# ReFS reference analyzer
+# By mmorsi - 2014-09-03
+
+require 'graphviz'
+
+CLUSTERS = [0x1e, 0x2e, 0x37, 0x38,
+            0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
+
+DISK     = 'flat-disk.vmdk'
+OFFSET   = 0x2100000
+
+inf  = File.open(DISK, 'rb')
+
+refs = {}
+
+CLUSTERS.each do |cluster|
+  offset = OFFSET + cluster * 0x4000
+  inf.seek(offset + 2) # skip first 2 bytes containing cluster #
+  while inf.pos < offset + 0x4000
+    checkb = inf.read(2).unpack('S').first
+    CLUSTERS.each do |checkc|
+      if checkb == checkc
+        refs[cluster]         ||= {}
+        refs[cluster][checkc] ||=  0
+        refs[cluster][checkc]  +=  1
+      end
+    end
+  end
+end
+
+inf.close
+
+g = GraphViz.new( :G, :type => :digraph )
+
+refs.keys.each { |cluster|
+  g.add_nodes cluster.to_s(16)
+}
+
+refs.keys.each { |cluster|
+  puts "cluster #{cluster.to_s(16)} references: "
+  refs[cluster].keys.each { |ref|
+    puts ref.to_s(16) + " (#{refs[cluster][ref]})"
+    #g.add_edges(cluster.to_s(16), ref.to_s(16))
+    g.add_edges(ref.to_s(16), cluster.to_s(16))
+  }
+}
+
+g.output :png => 'fan-out.png'

data/bin/rels.rb

@@ -0,0 +1,269 @@
+#!/usr/bin/ruby
+# ReFS File Lister
+# Copyright (C) 2014 Red Hat Inc.
+
+require 'optparse'
+require 'colored'
+
+FIRST_PAGE_ID =  0x1e
+PAGE_SIZE     =  0x4000
+FIRST_PAGE_ADDRESS = FIRST_PAGE_ID * PAGE_SIZE
+
+ROOT_DIR_ID   = [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0]
+DIR_ENTRY     = 0x20030
+FILE_ENTRY    = 0x10030
+
+DIR_TREE      = 0x301
+DIR_LIST      = 0x200
+#DIR_BRANCH    = 0x000 ?
+
+ADDRESSES  = {
+  # page
+  :virtual_page_number => 0x18,
+  :first_attr          => 0x30,
+
+  # page 0x1e
+  :system_table_page   => 0xA0,
+
+  # system table
+  :system_pages        => 0x58,
+
+  # generic table
+  :num_objects         => 0x20, # referenced from start of first attr
+
+  :table_length        => 0x04  # referenced from start of table header
+}
+
+def read_attribute(image)
+  pos      = image.pos
+  attr_len = image.read(4).unpack('L').first
+  return nil if attr_len == 0
+  image.seek pos
+
+  image.read(attr_len)
+end
+
+def record_boundries(attribute)
+  header        = attribute.unpack('S*')
+  key_offset    = header[2]
+  key_length    = header[3]
+  value_offset  = header[5]
+  value_length  = header[6]
+  [key_offset, key_length, value_offset, value_length]
+end
+
+def record_flags(attribute)
+  attribute.unpack('S*')[4]
+end
+
+def record_key(attribute)
+  ko, kl, vo, vl = record_boundries(attribute)
+  attribute.unpack('C*')[ko...ko+kl].pack('C*')
+end
+
+def record_value(attribute)
+  ko, kl, vo, vl = record_boundries(attribute)
+  attribute.unpack('C*')[vo..-1].pack('C*')
+end
+
+def filter_dir_record(dir_entry, opts)
+  image = opts[:image]
+
+  # '4' seems to indicate a historical record or similar,
+  # records w/ flags '0' or '8' are what we want
+  record_flags(dir_entry)== 4 ? filter_dir_record(read_attribute(opts[:image]), opts) : dir_entry 
+end
+
+def parse_dir_branch(node, prefix, opts)
+  key          = record_key(node)
+  value        = record_value(node)
+  flags        = record_flags(node)
+
+  value_dwords = value.unpack('L*')
+  value_qwords = value.unpack('Q*')
+
+  page_id      = value_dwords[0]
+  page_address = page_id * PAGE_SIZE
+  checksum     = value_qwords[2]
+  parse_dir_page page_address, prefix, opts unless checksum == 0 || flags == 4
+end
+
+def parse_dir_record(dir_entry, prefix, opts)
+  key        = record_key(dir_entry)
+  value      = record_value(dir_entry)
+
+  key_bytes  = key.unpack('C*')
+  key_dwords = key.unpack('L*')
+  entry_type = key_dwords.first
+  if entry_type == DIR_ENTRY
+    dir_name = key_bytes[4..-1].pack('L*')
+    opts[:dirs] << "#{prefix}\\#{dir_name}"
+
+    dir_obj = value.unpack('C*')[0...8]
+    dir_obj = [0, 0, 0, 0, 0, 0, 0, 0].concat(dir_obj)
+    parse_dir_obj(dir_obj, "#{prefix}\\#{dir_name}", opts)
+
+  elsif entry_type == FILE_ENTRY
+    file_name = key_bytes[4..-1].pack('L*')
+    opts[:files] << "#{prefix}\\#{file_name}"
+  end
+end
+
+def parse_dir_obj(object_id, prefix, opts)
+  image        = opts[:image]
+  object_pages = opts[:object_pages]
+  opts[:dirs]  ||= []
+  opts[:files] ||= []
+
+  page_id = object_pages[object_id]
+  page_address = page_id * PAGE_SIZE
+  parse_dir_page page_address, prefix, opts
+end
+
+def parse_dir_page(page_address, prefix, opts)
+  image        = opts[:image]
+  image_start  = opts[:offset]
+
+  # skip container/placeholder attribute
+  image.seek(image_start + page_address + ADDRESSES[:first_attr])
+  read_attribute(image)
+
+  # start of table attr, pull out table length, type
+  table_header_attr   = read_attribute(image)
+  table_header_dwords = table_header_attr.unpack("L*")
+  header_len          = table_header_dwords[0]
+  table_len           = table_header_dwords[1]
+  remaining_len       = table_len - header_len
+  table_type          = table_header_dwords[3]
+
+  until remaining_len == 0
+    orig_pos = image.pos
+    record   = read_attribute(image)
+
+    # need to keep track of position locally as we
+    # recursively call parse_dir via helpers
+    pos = image.pos
+
+    if table_type == DIR_TREE
+      parse_dir_branch record, prefix, opts
+    else #if table_type == DIR_LIST
+      record = filter_dir_record(record, opts)
+      pos = image.pos
+      parse_dir_record record, prefix, opts
+
+    end
+
+    image.seek pos
+    remaining_len -= (image.pos - orig_pos)
+  end
+end
+
+def parse_object_table(opts)
+  image       = opts[:image]
+  image_start = opts[:offset]
+  opts[:object_pages] ||= {}
+
+  # in the images I've seen this has always been the first entry
+  # in the system table, though always has virtual page id = 2
+  # which we could look for if this turns out not to be the case
+  object_page_id = opts[:system_pages].first
+  object_page_address = object_page_id * PAGE_SIZE
+
+  # read number of objects from index header
+  image.seek(image_start + object_page_address + ADDRESSES[:first_attr])
+  first_attr  = read_attribute(image)
+  num_objects = first_attr.unpack('L*')[ADDRESSES[:num_objects]/4]
+
+  # start of table attr, skip for now
+  read_attribute(image)
+
+  0.upto(num_objects-1) do
+    object_record  = read_attribute(image)
+    object_id      = record_key(object_record).unpack('C*')
+
+    # here object page is first qword of record value
+    object_page    = record_value(object_record).unpack('Q*').first
+    opts[:object_pages][object_id] = object_page
+  end
+end
+
+def parse_system_table(opts)
+  image       = opts[:image]
+  image_start = opts[:offset]
+  opts[:system_pages] ||= []
+
+  image.seek(image_start + FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
+  system_table_page = image.read(8).unpack('Q').first
+  system_table_address = system_table_page * PAGE_SIZE 
+
+  image.seek(image_start + system_table_address + ADDRESSES[:system_pages])
+  num_system_pages = image.read(4).unpack('L').first
+
+  0.upto(num_system_pages-1) do
+    system_page_offset = image.read(4).unpack('L').first
+    pos = image.pos
+
+    image.seek(image_start + system_table_address + system_page_offset)
+    system_page = image.read(8).unpack('Q').first
+    opts[:system_pages] << system_page
+
+    image.seek(pos)
+  end
+end
+
+def print_results(opts)
+  puts "Dirs found:".bold
+  opts[:dirs].each { |dir|
+    puts "#{dir}".blue
+  }
+
+  puts
+  puts "Files found:".bold
+  opts[:files].sort.each { |file|
+    puts "#{file}".red
+  }
+end
+
+def main(opts = {})
+  image = File.open(opts[:image], 'rb')
+  opts[:image] = image
+
+  parse_system_table(opts)
+  parse_object_table(opts)
+  parse_dir_obj(ROOT_DIR_ID, '', opts)
+  print_results(opts)
+end
+
+def parse_cli(cli)
+  opts   = {}
+  parser = OptionParser.new do |popts|
+    popts.on("-h", "--help", "Print help message") do
+      puts parser
+      exit
+    end
+
+    popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
+      opts[:image] = path
+    end
+
+    popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
+      opts[:offset] = offset.to_i
+    end
+  end
+
+  begin
+    parser.parse!(cli)
+  rescue OptionParser::InvalidOption
+    puts parser
+    exit
+  end
+
+  if !opts[:image] || !opts[:offset]
+    puts "--image and --offset params are needed at a minimum"
+    exit 1
+  end
+
+  opts
+end
+
+main parse_cli(ARGV) if __FILE__ == $0

data/bin/resilience.rb

@@ -0,0 +1,351 @@
+#!/usr/bin/ruby
+# resilience.rb - Ruby ReFS Parser
+
+require 'optparse'
+require 'colored'
+
+FIRST_PAGE_ID =  0x1e
+PAGE_SIZE     =  0x4000
+
+# note I believe we can also use the object-id's
+# of these entities
+ROOT_PAGE_NUMBER         = 0x00
+OBJECT_TABLE_PAGE_NUMBER = 0x02
+OBJECT_TREE_PAGE_NUMBER  = 0x03
+
+ADDRESSES  = {
+  # vbr
+  :bytes_per_sector    => 0x20,
+  :sectors_per_cluster => 0x24,
+
+  # page
+  :page_sequence       => 0x08, # shadow pages share the same virtual page number
+  :virtual_page_number => 0x18, # but will have higher sequences
+  :first_attr          => 0x30,
+
+  # index header
+  :object_id           => 0xC,  # possibly index type of similar
+  :entries_count       => 0x20
+}
+
+def unpack_attribute(image)
+  pos      = image.pos
+  attr_len = image.read(4).unpack('L').first
+  return nil if attr_len == 0
+  image.seek pos
+
+  image.read(attr_len).unpack('C*')
+end
+
+def process_attributes(image, start)
+  attributes = []
+  image.seek(start)
+  while true
+    attribute = unpack_attribute(image)
+    break if attribute.nil?
+    attributes << attribute
+  end
+
+  attributes
+end
+
+# Large object table seems to have a special edge case w/
+# an extra 0x40 data block, haven't deduced meaning of this yet
+def process_object_table_attributes(image, start)
+  image.seek(start)
+
+  attributes = []
+
+  # unpack first three attributes as normal
+  attributes << unpack_attribute(image)
+  attributes << unpack_attribute(image)
+  attributes << unpack_attribute(image)
+
+  # XXX hacky edge case detection, if next two bytes are 0,
+  # handling as extended block, skipping for now
+  if image.read(2).unpack('S').first == 0
+    image.seek(38, IO::SEEK_CUR)
+  else
+    image.seek(-2, IO::SEEK_CUR)
+  end
+
+  # process rest of attributes as normal
+  attributes + process_attributes(image, image.pos)
+end
+
+# Extract additional metadata from attributes
+def inspect_attributes(attributes)
+  return {} if attributes.empty?
+
+  object_id = attributes.first[ADDRESSES[:object_id]]
+  entries   = attributes.first[ADDRESSES[:entries_count]]
+  {:object_id => object_id, :entries => entries}
+end
+
+def parse_pages(data, opts)
+  image       = data[:image]
+  image_start = opts[:offset]
+
+  data[:pages].keys.each { |page|
+    page_offset = page * PAGE_SIZE
+
+    image.seek(image_start + page_offset + ADDRESSES[:page_sequence])
+    page_sequence = image.read(4).unpack('L').first
+
+    image.seek(image_start + page_offset + ADDRESSES[:virtual_page_number])
+    virtual_page_number = image.read(4).unpack('L').first
+
+    attributes_start = image_start + page_offset + ADDRESSES[:first_attr]
+
+    if virtual_page_number == ROOT_PAGE_NUMBER
+      # skipping root page analysis until it is further understood
+      is_root = true
+
+    elsif virtual_page_number == OBJECT_TABLE_PAGE_NUMBER
+      attributes = process_object_table_attributes(image, attributes_start)
+
+    else
+      attributes = process_attributes image, attributes_start
+    end
+
+    data[:pages][page][:sequence]            = page_sequence
+    data[:pages][page][:virtual_page_number] = virtual_page_number
+
+    unless is_root
+      data[:pages][page][:attributes]        = attributes
+      data[:pages][page].merge! inspect_attributes(attributes)
+    end
+  }
+end
+
+def volume_metadata(data, opts)
+  image       = data[:image]
+  image_start = opts[:offset]
+
+  image.seek(image_start + ADDRESSES[:bytes_per_sector])
+  bytes_per_sector = image.read(4).unpack('L').first
+
+  image.seek(image_start + ADDRESSES[:sectors_per_cluster])
+  sectors_per_cluster = image.read(4).unpack('L').first
+
+  cluster_size = bytes_per_sector * sectors_per_cluster
+
+  {:bytes_per_sector    => bytes_per_sector,
+   :sectors_per_cluster => sectors_per_cluster,
+   :cluster_size        => cluster_size }
+end
+
+def pages(data, opts)
+  image       = data[:image]
+  image_start = opts[:offset]
+  page        = FIRST_PAGE_ID
+  pages       = {}
+  
+  image.seek(image_start + page * PAGE_SIZE)
+  while contents = image.read(PAGE_SIZE)
+    # only pull out metadata pages currently
+    is_metadata = contents.unpack('S').first == page
+    pages[page] = {:contents => contents} if is_metadata
+
+    page += 1
+  end
+
+  pages
+end
+
+# Convert an array of bytes in little endian order to human friendly string
+def little_endian_str(bytes)
+  str = '0x'
+  value = false
+  bytes.reverse_each { |b|
+    next if b == 0 && !value
+    value = true
+    str += b.to_s(16)
+  }
+  str
+end
+
+def object_table_page_id(data)
+  # find shadow page w/ highest sequence
+  data[:pages].keys.select { |p| data[:pages][p][:virtual_page_number] == OBJECT_TABLE_PAGE_NUMBER }
+                   .sort   { |p1, p2| data[:pages][p2][:sequence] <=> data[:pages][p1][:sequence]  }.first
+end
+
+def object_table(data, opts)
+  table = {}
+  page  = data[:pages][object_table_page_id(data)]
+
+  # XXX this could start from the 2nd attribute if the exception in
+  #     process_table_attributes does _not_ apply, need to investigate furthur / fix
+  page[:attributes][3...-1].each { |bytes|
+    # bytes 4-7 give us the key offset & length and
+    key_offset = bytes[4..5].pack('C*').unpack('S').first.to_i
+    key_length = bytes[6..7].pack('C*').unpack('S').first.to_i
+
+    # bytes A-D give us the value offset & length
+    value_offset = bytes[0xA..0xB].pack('C*').unpack('S').first.to_i
+    value_length = bytes[0xC..0xD].pack('C*').unpack('S').first.to_i
+
+    key   = bytes[key_offset...key_offset+key_length]
+    value = bytes[value_offset...value_offset+value_length]
+
+    cluster_bytes = value[0..7]
+    # TODO extract 'type' from value[3a..3d]a (?)
+
+    object_id   = key.pack('C*')
+    cluster     = cluster_bytes.pack('C*')
+
+    object_str  = little_endian_str(key)
+    cluster_str = little_endian_str(cluster_bytes)
+
+    table[object_id] = {:object_str  => object_str,
+                        :cluster     => cluster,
+                        :cluster_str => cluster_str}
+  }
+  
+  table
+end
+
+def object_tree_page_id(data)
+  # find shadow page w/ highest sequence
+  data[:pages].keys.select { |p| data[:pages][p][:virtual_page_number] == OBJECT_TREE_PAGE_NUMBER }
+                   .sort   { |p1, p2| data[:pages][p2][:sequence] <=> data[:pages][p1][:sequence]  }.first
+end
+
+def object_tree(data, opts)
+  tree = {}
+  page  = data[:pages][object_tree_page_id(data)]
+
+  page[:attributes][2...-1].each { |bytes|
+    obj1_bytes = bytes[0x10..0x1F]
+    obj2_bytes = bytes[0x20..0x2F]
+
+    obj1 = little_endian_str(obj1_bytes)
+    obj2 = little_endian_str(obj2_bytes)
+
+    tree[obj1] ||= []
+    tree[obj1]  << obj2
+  }
+
+  tree
+end
+
+def data_str(data, str_opts = {})
+  places = str_opts[:places] || 1
+
+  return '0x'+ ('0' * places) if data.nil?
+  '0x'+data.to_s(16).rjust(places, '0').upcase
+end
+
+def print_results(data, opts)
+  out = "Analyzed ReFS filesystem on #{opts[:image].green.bold} "\
+        "starting at #{opts[:offset].to_s.green.bold}\n" \
+        "VBR: #{data_str(data[:bytes_per_sector]).to_s.yellow.bold} (bytes per sector) * " \
+        "#{data_str(data[:sectors_per_cluster]).to_s.yellow.bold} (sectors per cluster) = " \
+        "#{data_str(data[:cluster_size]).to_s.yellow.bold} (bytes per cluster)\n"
+
+  data[:pages].keys.each { |page_id|
+    page  = data[:pages][page_id]
+
+    page_out = "Page #{data_str(page_id, :places => 4).blue.bold}: "\
+               "number #{data_str(page[:virtual_page_number], :places => 3).blue.bold} - " \
+               "sequence #{data_str(page[:sequence], :places => 2).blue.bold} - " \
+               "object id #{data_str(page[:object_id], :places => 2).blue.bold} - " \
+               "records #{data_str(page[:entries], :places => 2).blue.bold}\n"
+
+    if opts[:attributes] && page[:attributes]
+      page_out += " Attributes:\n"
+      page[:attributes].each { |attr_values|
+        attr_out  = attr_values.collect { |a| a.to_s(16) }.join(' ')[0...10] +'...'
+        page_out += '  ' + attr_out + "\n"
+      }
+    end
+
+    out += page_out
+  }
+
+  if opts[:object_table]
+    out += "\nObject table:\n"
+    out += "Obj   | Cluster\n"
+    out += "-------------\n"
+    data[:object_table].keys.each { |obj_id|
+      object_str = data[:object_table][obj_id][:object_str]
+      cluster    = data[:object_table][obj_id][:cluster_str]
+      out += "#{object_str[0..4]} | #{cluster}\n"
+    }
+  end
+
+  if opts[:object_tree]
+    out += "\nObject tree:\n"
+    out += "-------------\n"
+    data[:object_tree].keys.each { |obj_id|
+      references = data[:object_tree][obj_id].collect { |obj| obj[0..4] }.join(', ')
+      out += "#{obj_id[0..4]} -> #{references}\n"
+    }
+  end
+
+  puts out
+end
+
+def main(opts = {})
+  image = File.open(opts[:image], 'rb')
+
+  data = {}
+  data[:image] = image
+
+  data.merge! volume_metadata(data, opts)
+  data.merge! :pages         => pages(data, opts)
+
+  parse_pages data, opts
+
+  data.merge! :object_table  => object_table(data, opts)
+  data.merge! :object_tree   => object_tree(data, opts)
+
+  print_results data, opts
+end
+
+def parse_cli(cli)
+  opts   = {}
+  parser = OptionParser.new do |popts|
+    popts.on("-h", "--help", "Print help message") do
+      puts parser
+      exit
+    end
+
+    popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
+      opts[:image] = path
+    end
+
+    popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
+      opts[:offset] = offset.to_i
+    end
+
+    popts.on("-a", "--attributes", "Include attribute analysis in output") do
+      opts[:attributes] = true
+    end
+
+    popts.on("--table", "Include object table analysis in output") do
+      opts[:object_table] = true
+    end
+
+    popts.on("--tree", "Include object tree analysis in output") do
+      opts[:object_tree] = true
+    end
+  end
+
+  begin
+    parser.parse!(cli)
+  rescue OptionParser::InvalidOption
+    puts parser
+    exit
+  end
+
+  if !opts[:image] || !opts[:offset]
+    puts "--image and --offset params are needed at a minimum"
+    exit 1
+  end
+
+  opts
+end
+
+main parse_cli(ARGV) if __FILE__ == $0

data/bin/rex.rb

@@ -0,0 +1,74 @@
+#!/usr/bin/ruby
+# ReFS File Extractor
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'optparse'
+require 'resilience'
+
+def main
+  cli_opts = parse_cli(ARGV)
+  results  = parse_image cli_opts
+  write_results cli_opts, results
+end
+
+def write_results(opts, results)
+  dir = opts[:dir]
+  Dir.mkdir(dir) unless File.directory?(dir)
+
+  results[:files].each do |name, contents|
+    puts "Got #{name}"
+    path = "#{dir}/#{name}".delete("\0")
+    File.write(path, contents)
+  end
+end
+
+def parse_image(opts)
+  file         = File.open(opts[:image], 'rb')
+  image        = Resilience::OnImage.image
+  image.file   = file
+  image.offset = opts[:offset]
+  image.opts   = opts
+
+  image.parse
+  files        = image.root_dir.files
+  dirs         = image.root_dir.dirs
+  {:files => files, :dirs => dirs}
+end
+
+def parse_cli(cli)
+  opts   = {}
+  parser = OptionParser.new do |popts|
+    popts.on("-h", "--help", "Print help message") do
+      puts parser
+      exit
+    end
+
+    popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
+      opts[:image] = path
+    end
+
+    popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
+      opts[:offset] = offset.to_i
+    end
+
+    popts.on("-d", "--dir dir", "Output directory") do |dir|
+      opts[:dir] = dir
+    end
+  end
+
+  begin
+    parser.parse!(cli)
+  rescue OptionParser::InvalidOption
+    puts parser
+    exit
+  end
+
+  if !opts[:image] || !opts[:offset] || !opts[:dir]
+    puts "--image, --offset, and --dir params are needed"
+    exit 1
+  end
+
+  opts
+end
+
+main if __FILE__ == $0

data/lib/resilience.rb

@@ -0,0 +1,13 @@
+#!/usr/bin/ruby
+# ReFS Parser (expiremental)
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/mixins'
+require 'resilience/constants'
+
+require 'resilience/fs_dir'
+require 'resilience/attribute'
+require 'resilience/image'
+
+require 'resilience/dirs'
+require 'resilience/tables'

data/lib/resilience/attribute.rb

@@ -0,0 +1,28 @@
+#!/usr/bin/ruby
+# ReFS Attributes
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class Attribute
+    include OnImage
+
+    attr_accessor :bytes
+
+    def initialize(args={})
+      @bytes = args[:bytes] if args.key?(:bytes)
+    end
+
+    def self.read
+      pos      = image.pos
+      attr_len = image.read(4).unpack('L').first
+      return new if attr_len == 0
+
+      image.seek pos
+      new(:bytes => image.read(attr_len))
+    end
+
+    def unpack(format)
+      bytes.unpack(format)
+    end
+  end
+end # module Resilience

data/lib/resilience/constants.rb

@@ -0,0 +1,32 @@
+#!/usr/bin/ruby
+# ReFS Constants
+# Copyright (C) 2015 Red Hat Inc.
+
+FIRST_PAGE_ID =  0x1e
+PAGE_SIZE     =  0x4000
+FIRST_PAGE_ADDRESS = FIRST_PAGE_ID * PAGE_SIZE
+
+ROOT_DIR_ID   = [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0]
+DIR_ENTRY     = 0x20030
+FILE_ENTRY    = 0x10030
+
+DIR_TREE      = 0x301
+DIR_LIST      = 0x200
+#DIR_BRANCH    = 0x000 ?
+
+ADDRESSES  = {
+  # page
+  :virtual_page_number => 0x18,
+  :first_attr          => 0x30,
+
+  # page 0x1e
+  :system_table_page   => 0xA0,
+
+  # system table
+  :system_pages        => 0x58,
+
+  # generic table
+  :num_objects         => 0x20, # referenced from start of first attr
+
+  :table_length        => 0x04  # referenced from start of table header
+}

data/lib/resilience/dirs.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby
+# ReFS Directories
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/dirs/root_dir'

data/lib/resilience/dirs/root_dir.rb

@@ -0,0 +1,15 @@
+#!/usr/bin/ruby
+# ReFS Root Dir
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class RootDir < FSDir::DirBase
+    include OnImage
+
+    def self.parse
+      dir = new
+      dir.parse_dir_obj ROOT_DIR_ID, ''
+      dir
+    end
+  end  # class RootDir
+end # module Resilience

data/lib/resilience/fs_dir.rb

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# ReFS FileSystem Directory
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/fs_dir/dir_base'
+require 'resilience/fs_dir/record'

data/lib/resilience/fs_dir/dir_base.rb

@@ -0,0 +1,106 @@
+#!/usr/bin/ruby
+# ReFS Directory Handling
+# Copyright (C) 2014-2015 Red Hat Inc.
+
+require 'fileutils'
+
+module Resilience
+  module FSDir
+    class DirBase
+      include OnImage
+
+      attr_accessor :dirs
+      attr_accessor :files
+
+      def parse_dir_obj(object_id, prefix)
+        object_table   = image.object_table
+        @dirs        ||= {}
+        @files       ||= {}
+      
+        page_id      = object_table.pages[object_id]
+        page_address = page_id * PAGE_SIZE
+        parse_dir_page page_address, prefix
+      end
+      
+      def parse_dir_page(page_address, prefix)
+        # skip container/placeholder attribute
+        image.seek(page_address + ADDRESSES[:first_attr])
+        Attribute.read
+      
+        # start of table attr, pull out table length, type
+        table_header_attr   = Attribute.read
+        table_header_dwords = table_header_attr.unpack("L*")
+        header_len          = table_header_dwords[0]
+        table_len           = table_header_dwords[1]
+        remaining_len       = table_len - header_len
+        table_type          = table_header_dwords[3]
+      
+        until remaining_len == 0
+          orig_pos = image.pos
+          record   = Record.read
+      
+          # need to keep track of position locally as we
+          # recursively call parse_dir via helpers
+          pos = image.pos
+      
+          if table_type == DIR_TREE
+            parse_dir_branch record, prefix
+      
+          else #if table_type == DIR_LIST
+            record = filter_dir_record(record)
+            pos = image.pos
+            parse_dir_record record, prefix
+      
+          end
+      
+          image.seek pos
+          remaining_len -= (image.pos - orig_pos)
+        end
+      end
+
+      def filter_dir_record(record)
+        # '4' seems to indicate a historical record or similar,
+        # records w/ flags '0' or '8' are what we want
+        record.flags == 4 ? filter_dir_record(Record.read) : record 
+      end
+      
+      def parse_dir_branch(record, prefix)
+        key          = record.key
+        value        = record.value
+        flags        = record.flags
+      
+        value_dwords = value.unpack('L*')
+        value_qwords = value.unpack('Q*')
+      
+        page_id      = value_dwords[0]
+        page_address = page_id * PAGE_SIZE
+        checksum     = value_qwords[2]
+
+        parse_dir_page page_address, prefix unless checksum == 0 || flags == 4
+      end
+      
+      def parse_dir_record(record, prefix)
+        key        = record.key
+        value      = record.value
+      
+        key_bytes  = key.unpack('C*')
+        key_dwords = key.unpack('L*')
+        entry_type = key_dwords.first
+
+        if entry_type == DIR_ENTRY
+          dir_name = key_bytes[4..-1].pack('L*')
+          dir_obj = value.unpack('C*')[0...8]
+          dirs["#{prefix}\\#{dir_name}"] = dir_obj
+
+          dir_obj = [0, 0, 0, 0, 0, 0, 0, 0].concat(dir_obj)
+          parse_dir_obj(dir_obj, "#{prefix}\\#{dir_name}")
+      
+        elsif entry_type == FILE_ENTRY
+          file_name = key_bytes[4..-1].pack('L*')
+          prefixed  = "#{prefix}\\#{file_name}"
+          files[prefixed] = value
+        end
+      end
+    end # class DirBase
+  end # module FSDir
+end # module Resilience

data/lib/resilience/fs_dir/record.rb

@@ -0,0 +1,59 @@
+#!/usr/bin/ruby
+# ReFS Records
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  module FSDir
+    class Record
+      include OnImage
+  
+      attr_accessor :attribute
+  
+      def initialize(attribute)
+        @attribute = attribute
+      end
+  
+      def self.read
+        new(Attribute.read)
+      end
+  
+      def calc_boundries
+        return if @boundries_set
+        @boundries_set = true
+  
+        header         = attribute.unpack('S*')
+        @key_offset    = header[2]
+        @key_length    = header[3]
+        @value_offset  = header[5]
+        @value_length  = header[6]
+      end
+  
+      def boundries
+        calc_boundries
+        [@key_offset, @key_length, @value_offset, @value_length]
+      end
+  
+      def calc_flags
+        return if @flags_set
+        @flags_set = true
+  
+        @flags = attribute.unpack('S*')[4]
+      end
+  
+      def flags
+        calc_flags
+        @flags
+      end
+  
+      def key
+        ko, kl, vo, vl = boundries
+        attribute.unpack('C*')[ko...ko+kl].pack('C*')
+      end
+  
+      def value
+        ko, kl, vo, vl = boundries
+        attribute.unpack('C*')[vo..-1].pack('C*')
+      end
+    end # class Record
+  end # module FSDir
+end # module Resilience

data/lib/resilience/image.rb

@@ -0,0 +1,37 @@
+#!/usr/bin/ruby
+# ReFS Image Representation
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class Image
+    attr_accessor :file
+    attr_accessor :offset
+    attr_accessor :opts
+
+    attr_accessor :root_dir
+    attr_accessor :system_table
+    attr_accessor :object_table
+
+    def initialize(args={})
+      @file         = args[:file] || []
+    end
+
+    def parse
+      @system_table = Resilience::SystemTable.parse
+      @object_table = Resilience::ObjectTable.parse
+      @root_dir     = Resilience::RootDir.parse
+    end
+
+    def seek(position)
+      @file.seek offset + position
+    end
+
+    def pos
+      @file.pos - offset
+    end
+
+    def read(len)
+      @file.read(len)
+    end
+  end
+end # module Resilience

data/lib/resilience/mixins.rb

@@ -0,0 +1,5 @@
+#!/usr/bin/ruby
+# Resilience Mixins
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/mixins/on_image'

data/lib/resilience/mixins/on_image.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/ruby
+# ReFS On Image Mixin
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  module OnImage
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def self.image
+      @image ||= Resilience::Image.new
+    end
+
+    def image
+      OnImage.image
+    end
+
+    module ClassMethods
+      def image
+        OnImage.image
+      end
+    end
+  end # module OnImage
+end # module Resilience

data/lib/resilience/tables.rb

@@ -0,0 +1,6 @@
+#!/usr/bin/ruby
+# ReFS Tables
+# Copyright (C) 2015 Red Hat Inc.
+
+require 'resilience/tables/object'
+require 'resilience/tables/system'

data/lib/resilience/tables/object.rb

@@ -0,0 +1,46 @@
+#!/usr/bin/ruby
+# ReFS Object Table
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class ObjectTable
+    include OnImage
+
+    attr_accessor :pages
+
+    def initialize
+      @pages ||= {}
+    end
+
+    def self.parse
+      table = new
+      table.parse_pages
+      table
+    end
+
+    def parse_pages
+      # in the images I've seen this has always been the first entry
+      # in the system table, though always has virtual page id = 2
+      # which we could look for if this turns out not to be the case
+      object_page_id      = image.system_table.pages.first
+      object_page_address = object_page_id * PAGE_SIZE
+
+      # read number of objects from index header
+      image.seek(object_page_address + ADDRESSES[:first_attr])
+      first_attr  = Attribute.read
+      num_objects = first_attr.unpack('L*')[ADDRESSES[:num_objects]/4]
+
+      # start of table attr, skip for now
+      Attribute.read
+
+      0.upto(num_objects-1) do
+        object_record     = FSDir::Record.read
+        object_id         = object_record.key.unpack('C*')
+
+        # here object page is first qword of record value
+        object_page       = object_record.value.unpack('Q*').first
+        @pages[object_id] = object_page
+      end
+    end
+  end # class ObjectTable
+end # module Resilience

data/lib/resilience/tables/system.rb

@@ -0,0 +1,42 @@
+#!/usr/bin/ruby
+# ReFS System Table
+# Copyright (C) 2015 Red Hat Inc.
+
+module Resilience
+  class SystemTable
+    include OnImage
+
+    attr_accessor :pages
+
+    def initialize
+      @pages = []
+    end
+
+    def self.parse
+      table = new
+      table.parse_pages
+      table
+    end
+
+    def parse_pages
+      image.seek(FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
+      system_table_page    = image.read(8).unpack('Q').first
+      system_table_address = system_table_page * PAGE_SIZE 
+
+      image.seek(system_table_address + ADDRESSES[:system_pages])
+      num_system_pages = image.read(4).unpack('L').first
+
+      0.upto(num_system_pages-1) do
+        system_page_offset = image.read(4).unpack('L').first
+        pos = image.pos
+
+        image.seek(system_table_address + system_page_offset)
+        system_page = image.read(8).unpack('Q').first
+        @pages << system_page
+
+        image.seek(pos)
+      end
+    end
+  end # class SystemTable
+end # module Resilience
+

metadata

@@ -0,0 +1,72 @@
+--- !ruby/object:Gem::Specification
+name: resilience
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Mo Morsi
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-05-23 00:00:00.000000000 Z
+dependencies: []
+description: Ruby ReFS utils
+email: mmorsi@redhat.com
+executables:
+- fs.rb
+- resilience.rb
+- rex.rb
+- rels.rb
+- ref.rb
+- clum.py
+- cle.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- README.md
+- bin/cle.rb
+- bin/clum.py
+- bin/fs.rb
+- bin/ref.rb
+- bin/rels.rb
+- bin/resilience.rb
+- bin/rex.rb
+- lib/resilience.rb
+- lib/resilience/attribute.rb
+- lib/resilience/constants.rb
+- lib/resilience/dirs.rb
+- lib/resilience/dirs/root_dir.rb
+- lib/resilience/fs_dir.rb
+- lib/resilience/fs_dir/dir_base.rb
+- lib/resilience/fs_dir/record.rb
+- lib/resilience/image.rb
+- lib/resilience/mixins.rb
+- lib/resilience/mixins/on_image.rb
+- lib/resilience/tables.rb
+- lib/resilience/tables/object.rb
+- lib/resilience/tables/system.rb
+homepage: https://gist.github.com/movitto/866de4356f56a3b478ca
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A module/command-line utility to parse a ReFS file system image
+test_files: []