resilience 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,7 @@
1
+ ---
2
+ SHA1:
3
+ metadata.gz: 4ce546fb9a6dcfd401ccb8adbfa16bb3147f3a3b
4
+ data.tar.gz: 6c47f4c2c0c3f89d74a30335928ef8bb991dcd0b
5
+ SHA512:
6
+ metadata.gz: 7d25ff402e660223d44333beab243b7281d8382a8a2f31caeea6670e1adef46fc6fd54d218475ecd2b0f7061148b2f0111b297e45ffcbc412886920dbcf23bee
7
+ data.tar.gz: 9767ac37ea16d858c02d89a217d33a8959509f03c1e7974d8b37d81db05b50576b8501a69ea1392338afafff7a06fae8c8a58dee3418198af1625d5120a1d199
@@ -0,0 +1,3 @@
1
+ Resilience - Experimental ReFS Library
2
+ Copyright (C) 2014-2015 Red Hat Inc.
3
+ Licensed under the MIT license
@@ -0,0 +1,24 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS 0x4000 Cluster Extractor
3
+ # By mmorsi - 2014-07-14
4
+
5
+ CLUSTERS = [0x1e, 0x20, 0x21, 0x22, 0x28, 0x29, 0x2a, 0x2b, 0x2c, 0x2d, 0x2e, 0x2f, 0x30, 0x31, 0x32, 0x33, 0x34, 0x35, 0x36, 0x37, 0x38,
6
+ 0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
7
+
8
+ DISK = 'flat-disk.vmdk'
9
+ OUTDIR = 'clusters'
10
+
11
+ REFS_VOLUME_OFFSET = 0x2100000
12
+
13
+ inf = File.open(DISK, 'rb')
14
+ CLUSTERS.each do |cluster|
15
+ outf = File.open("#{OUTDIR}/#{cluster.to_s(16)}", 'wb')
16
+
17
+ offset = REFS_VOLUME_OFFSET + cluster * 0x4000
18
+ inf.seek(offset)
19
+ contents = inf.read(0x4000)
20
+ outf.write contents
21
+ outf.close
22
+ end
23
+
24
+ inf.close
@@ -0,0 +1,28 @@
1
+ # ReFS 0x4000 Cluster Usage Mapper
2
+ # By Willi Ballenthin 2012-03-25
3
+ # With modifications by mmorsi - 2014-07-14
4
+ import sys, struct
5
+
6
+ #REFS_VOLUME_OFFSET = 0x2010000
7
+ REFS_VOLUME_OFFSET = 0x2100000
8
+
9
+ def main(args):
10
+ with open(args[1], "rb") as f:
11
+ global REFS_VOLUME_OFFSET
12
+ offset = REFS_VOLUME_OFFSET
13
+ cluster = 0
14
+ while True:
15
+ f.seek(offset + cluster * 0x4000)
16
+ buf = f.read(4)
17
+ if not buf: break
18
+ magic = struct.unpack("<I", buf)[0]
19
+ if magic == cluster:
20
+ print "Metadata cluster %s (%s)" % \
21
+ (hex(cluster), hex(offset + cluster * 0x4000))
22
+ elif magic != 0:
23
+ print "Non-null cluster %s (%s)" % \
24
+ (hex(cluster), hex(offset + cluster * 0x4000))
25
+ cluster += 1
26
+
27
+ if __name__ == '__main__':
28
+ main(sys.argv)
@@ -0,0 +1,21 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS file searcher
3
+ # By mmorsi - 2014-09-03
4
+
5
+ DISK = 'flat-disk.vmdk'
6
+ OFFSET = 0x2100000
7
+
8
+ SEQUENCE_LENGTH = 8
9
+ SEQUENCE = 0xe010002800000038 # inverted due to endian ordering
10
+
11
+ inf = File.open(DISK, 'rb')
12
+ inf.seek(OFFSET)
13
+
14
+ while check = inf.read(SEQUENCE_LENGTH)
15
+ check = check.unpack('Q').first
16
+ if check == SEQUENCE
17
+ puts 'File at: 0x' + inf.pos.to_s(16) + ' cluster ' + ((inf.pos - OFFSET) / 0x4000).to_s(16)
18
+ end
19
+ end
20
+
21
+ inf.close
@@ -0,0 +1,49 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS reference analyzer
3
+ # By mmorsi - 2014-09-03
4
+
5
+ require 'graphviz'
6
+
7
+ CLUSTERS = [0x1e, 0x2e, 0x37, 0x38,
8
+ 0x2c0, 0x2c1, 0x2c2, 0x2c3, 0x2c4, 0x2c5, 0x2c6, 0x2c7, 0x2c8, 0x2cc, 0x2cd, 0x2ce, 0x2cf]
9
+
10
+ DISK = 'flat-disk.vmdk'
11
+ OFFSET = 0x2100000
12
+
13
+ inf = File.open(DISK, 'rb')
14
+
15
+ refs = {}
16
+
17
+ CLUSTERS.each do |cluster|
18
+ offset = OFFSET + cluster * 0x4000
19
+ inf.seek(offset + 2) # skip first 2 bytes containing cluster #
20
+ while inf.pos < offset + 0x4000
21
+ checkb = inf.read(2).unpack('S').first
22
+ CLUSTERS.each do |checkc|
23
+ if checkb == checkc
24
+ refs[cluster] ||= {}
25
+ refs[cluster][checkc] ||= 0
26
+ refs[cluster][checkc] += 1
27
+ end
28
+ end
29
+ end
30
+ end
31
+
32
+ inf.close
33
+
34
+ g = GraphViz.new( :G, :type => :digraph )
35
+
36
+ refs.keys.each { |cluster|
37
+ g.add_nodes cluster.to_s(16)
38
+ }
39
+
40
+ refs.keys.each { |cluster|
41
+ puts "cluster #{cluster.to_s(16)} references: "
42
+ refs[cluster].keys.each { |ref|
43
+ puts ref.to_s(16) + " (#{refs[cluster][ref]})"
44
+ #g.add_edges(cluster.to_s(16), ref.to_s(16))
45
+ g.add_edges(ref.to_s(16), cluster.to_s(16))
46
+ }
47
+ }
48
+
49
+ g.output :png => 'fan-out.png'
@@ -0,0 +1,269 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS File Lister
3
+ # Copyright (C) 2014 Red Hat Inc.
4
+
5
+ require 'optparse'
6
+ require 'colored'
7
+
8
+ FIRST_PAGE_ID = 0x1e
9
+ PAGE_SIZE = 0x4000
10
+ FIRST_PAGE_ADDRESS = FIRST_PAGE_ID * PAGE_SIZE
11
+
12
+ ROOT_DIR_ID = [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0]
13
+ DIR_ENTRY = 0x20030
14
+ FILE_ENTRY = 0x10030
15
+
16
+ DIR_TREE = 0x301
17
+ DIR_LIST = 0x200
18
+ #DIR_BRANCH = 0x000 ?
19
+
20
+ ADDRESSES = {
21
+ # page
22
+ :virtual_page_number => 0x18,
23
+ :first_attr => 0x30,
24
+
25
+ # page 0x1e
26
+ :system_table_page => 0xA0,
27
+
28
+ # system table
29
+ :system_pages => 0x58,
30
+
31
+ # generic table
32
+ :num_objects => 0x20, # referenced from start of first attr
33
+
34
+ :table_length => 0x04 # referenced from start of table header
35
+ }
36
+
37
+ def read_attribute(image)
38
+ pos = image.pos
39
+ attr_len = image.read(4).unpack('L').first
40
+ return nil if attr_len == 0
41
+ image.seek pos
42
+
43
+ image.read(attr_len)
44
+ end
45
+
46
+ def record_boundries(attribute)
47
+ header = attribute.unpack('S*')
48
+ key_offset = header[2]
49
+ key_length = header[3]
50
+ value_offset = header[5]
51
+ value_length = header[6]
52
+ [key_offset, key_length, value_offset, value_length]
53
+ end
54
+
55
+ def record_flags(attribute)
56
+ attribute.unpack('S*')[4]
57
+ end
58
+
59
+ def record_key(attribute)
60
+ ko, kl, vo, vl = record_boundries(attribute)
61
+ attribute.unpack('C*')[ko...ko+kl].pack('C*')
62
+ end
63
+
64
+ def record_value(attribute)
65
+ ko, kl, vo, vl = record_boundries(attribute)
66
+ attribute.unpack('C*')[vo..-1].pack('C*')
67
+ end
68
+
69
+ def filter_dir_record(dir_entry, opts)
70
+ image = opts[:image]
71
+
72
+ # '4' seems to indicate a historical record or similar,
73
+ # records w/ flags '0' or '8' are what we want
74
+ record_flags(dir_entry)== 4 ? filter_dir_record(read_attribute(opts[:image]), opts) : dir_entry
75
+ end
76
+
77
+ def parse_dir_branch(node, prefix, opts)
78
+ key = record_key(node)
79
+ value = record_value(node)
80
+ flags = record_flags(node)
81
+
82
+ value_dwords = value.unpack('L*')
83
+ value_qwords = value.unpack('Q*')
84
+
85
+ page_id = value_dwords[0]
86
+ page_address = page_id * PAGE_SIZE
87
+ checksum = value_qwords[2]
88
+ parse_dir_page page_address, prefix, opts unless checksum == 0 || flags == 4
89
+ end
90
+
91
+ def parse_dir_record(dir_entry, prefix, opts)
92
+ key = record_key(dir_entry)
93
+ value = record_value(dir_entry)
94
+
95
+ key_bytes = key.unpack('C*')
96
+ key_dwords = key.unpack('L*')
97
+ entry_type = key_dwords.first
98
+ if entry_type == DIR_ENTRY
99
+ dir_name = key_bytes[4..-1].pack('L*')
100
+ opts[:dirs] << "#{prefix}\\#{dir_name}"
101
+
102
+ dir_obj = value.unpack('C*')[0...8]
103
+ dir_obj = [0, 0, 0, 0, 0, 0, 0, 0].concat(dir_obj)
104
+ parse_dir_obj(dir_obj, "#{prefix}\\#{dir_name}", opts)
105
+
106
+ elsif entry_type == FILE_ENTRY
107
+ file_name = key_bytes[4..-1].pack('L*')
108
+ opts[:files] << "#{prefix}\\#{file_name}"
109
+ end
110
+ end
111
+
112
+ def parse_dir_obj(object_id, prefix, opts)
113
+ image = opts[:image]
114
+ object_pages = opts[:object_pages]
115
+ opts[:dirs] ||= []
116
+ opts[:files] ||= []
117
+
118
+ page_id = object_pages[object_id]
119
+ page_address = page_id * PAGE_SIZE
120
+ parse_dir_page page_address, prefix, opts
121
+ end
122
+
123
+ def parse_dir_page(page_address, prefix, opts)
124
+ image = opts[:image]
125
+ image_start = opts[:offset]
126
+
127
+ # skip container/placeholder attribute
128
+ image.seek(image_start + page_address + ADDRESSES[:first_attr])
129
+ read_attribute(image)
130
+
131
+ # start of table attr, pull out table length, type
132
+ table_header_attr = read_attribute(image)
133
+ table_header_dwords = table_header_attr.unpack("L*")
134
+ header_len = table_header_dwords[0]
135
+ table_len = table_header_dwords[1]
136
+ remaining_len = table_len - header_len
137
+ table_type = table_header_dwords[3]
138
+
139
+ until remaining_len == 0
140
+ orig_pos = image.pos
141
+ record = read_attribute(image)
142
+
143
+ # need to keep track of position locally as we
144
+ # recursively call parse_dir via helpers
145
+ pos = image.pos
146
+
147
+ if table_type == DIR_TREE
148
+ parse_dir_branch record, prefix, opts
149
+ else #if table_type == DIR_LIST
150
+ record = filter_dir_record(record, opts)
151
+ pos = image.pos
152
+ parse_dir_record record, prefix, opts
153
+
154
+ end
155
+
156
+ image.seek pos
157
+ remaining_len -= (image.pos - orig_pos)
158
+ end
159
+ end
160
+
161
+ def parse_object_table(opts)
162
+ image = opts[:image]
163
+ image_start = opts[:offset]
164
+ opts[:object_pages] ||= {}
165
+
166
+ # in the images I've seen this has always been the first entry
167
+ # in the system table, though always has virtual page id = 2
168
+ # which we could look for if this turns out not to be the case
169
+ object_page_id = opts[:system_pages].first
170
+ object_page_address = object_page_id * PAGE_SIZE
171
+
172
+ # read number of objects from index header
173
+ image.seek(image_start + object_page_address + ADDRESSES[:first_attr])
174
+ first_attr = read_attribute(image)
175
+ num_objects = first_attr.unpack('L*')[ADDRESSES[:num_objects]/4]
176
+
177
+ # start of table attr, skip for now
178
+ read_attribute(image)
179
+
180
+ 0.upto(num_objects-1) do
181
+ object_record = read_attribute(image)
182
+ object_id = record_key(object_record).unpack('C*')
183
+
184
+ # here object page is first qword of record value
185
+ object_page = record_value(object_record).unpack('Q*').first
186
+ opts[:object_pages][object_id] = object_page
187
+ end
188
+ end
189
+
190
+ def parse_system_table(opts)
191
+ image = opts[:image]
192
+ image_start = opts[:offset]
193
+ opts[:system_pages] ||= []
194
+
195
+ image.seek(image_start + FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
196
+ system_table_page = image.read(8).unpack('Q').first
197
+ system_table_address = system_table_page * PAGE_SIZE
198
+
199
+ image.seek(image_start + system_table_address + ADDRESSES[:system_pages])
200
+ num_system_pages = image.read(4).unpack('L').first
201
+
202
+ 0.upto(num_system_pages-1) do
203
+ system_page_offset = image.read(4).unpack('L').first
204
+ pos = image.pos
205
+
206
+ image.seek(image_start + system_table_address + system_page_offset)
207
+ system_page = image.read(8).unpack('Q').first
208
+ opts[:system_pages] << system_page
209
+
210
+ image.seek(pos)
211
+ end
212
+ end
213
+
214
+ def print_results(opts)
215
+ puts "Dirs found:".bold
216
+ opts[:dirs].each { |dir|
217
+ puts "#{dir}".blue
218
+ }
219
+
220
+ puts
221
+ puts "Files found:".bold
222
+ opts[:files].sort.each { |file|
223
+ puts "#{file}".red
224
+ }
225
+ end
226
+
227
+ def main(opts = {})
228
+ image = File.open(opts[:image], 'rb')
229
+ opts[:image] = image
230
+
231
+ parse_system_table(opts)
232
+ parse_object_table(opts)
233
+ parse_dir_obj(ROOT_DIR_ID, '', opts)
234
+ print_results(opts)
235
+ end
236
+
237
+ def parse_cli(cli)
238
+ opts = {}
239
+ parser = OptionParser.new do |popts|
240
+ popts.on("-h", "--help", "Print help message") do
241
+ puts parser
242
+ exit
243
+ end
244
+
245
+ popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
246
+ opts[:image] = path
247
+ end
248
+
249
+ popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
250
+ opts[:offset] = offset.to_i
251
+ end
252
+ end
253
+
254
+ begin
255
+ parser.parse!(cli)
256
+ rescue OptionParser::InvalidOption
257
+ puts parser
258
+ exit
259
+ end
260
+
261
+ if !opts[:image] || !opts[:offset]
262
+ puts "--image and --offset params are needed at a minimum"
263
+ exit 1
264
+ end
265
+
266
+ opts
267
+ end
268
+
269
+ main parse_cli(ARGV) if __FILE__ == $0
@@ -0,0 +1,351 @@
1
+ #!/usr/bin/ruby
2
+ # resilience.rb - Ruby ReFS Parser
3
+
4
+ require 'optparse'
5
+ require 'colored'
6
+
7
+ FIRST_PAGE_ID = 0x1e
8
+ PAGE_SIZE = 0x4000
9
+
10
+ # note I believe we can also use the object-id's
11
+ # of these entities
12
+ ROOT_PAGE_NUMBER = 0x00
13
+ OBJECT_TABLE_PAGE_NUMBER = 0x02
14
+ OBJECT_TREE_PAGE_NUMBER = 0x03
15
+
16
+ ADDRESSES = {
17
+ # vbr
18
+ :bytes_per_sector => 0x20,
19
+ :sectors_per_cluster => 0x24,
20
+
21
+ # page
22
+ :page_sequence => 0x08, # shadow pages share the same virtual page number
23
+ :virtual_page_number => 0x18, # but will have higher sequences
24
+ :first_attr => 0x30,
25
+
26
+ # index header
27
+ :object_id => 0xC, # possibly index type of similar
28
+ :entries_count => 0x20
29
+ }
30
+
31
+ def unpack_attribute(image)
32
+ pos = image.pos
33
+ attr_len = image.read(4).unpack('L').first
34
+ return nil if attr_len == 0
35
+ image.seek pos
36
+
37
+ image.read(attr_len).unpack('C*')
38
+ end
39
+
40
+ def process_attributes(image, start)
41
+ attributes = []
42
+ image.seek(start)
43
+ while true
44
+ attribute = unpack_attribute(image)
45
+ break if attribute.nil?
46
+ attributes << attribute
47
+ end
48
+
49
+ attributes
50
+ end
51
+
52
+ # Large object table seems to have a special edge case w/
53
+ # an extra 0x40 data block, haven't deduced meaning of this yet
54
+ def process_object_table_attributes(image, start)
55
+ image.seek(start)
56
+
57
+ attributes = []
58
+
59
+ # unpack first three attributes as normal
60
+ attributes << unpack_attribute(image)
61
+ attributes << unpack_attribute(image)
62
+ attributes << unpack_attribute(image)
63
+
64
+ # XXX hacky edge case detection, if next two bytes are 0,
65
+ # handling as extended block, skipping for now
66
+ if image.read(2).unpack('S').first == 0
67
+ image.seek(38, IO::SEEK_CUR)
68
+ else
69
+ image.seek(-2, IO::SEEK_CUR)
70
+ end
71
+
72
+ # process rest of attributes as normal
73
+ attributes + process_attributes(image, image.pos)
74
+ end
75
+
76
+ # Extract additional metadata from attributes
77
+ def inspect_attributes(attributes)
78
+ return {} if attributes.empty?
79
+
80
+ object_id = attributes.first[ADDRESSES[:object_id]]
81
+ entries = attributes.first[ADDRESSES[:entries_count]]
82
+ {:object_id => object_id, :entries => entries}
83
+ end
84
+
85
+ def parse_pages(data, opts)
86
+ image = data[:image]
87
+ image_start = opts[:offset]
88
+
89
+ data[:pages].keys.each { |page|
90
+ page_offset = page * PAGE_SIZE
91
+
92
+ image.seek(image_start + page_offset + ADDRESSES[:page_sequence])
93
+ page_sequence = image.read(4).unpack('L').first
94
+
95
+ image.seek(image_start + page_offset + ADDRESSES[:virtual_page_number])
96
+ virtual_page_number = image.read(4).unpack('L').first
97
+
98
+ attributes_start = image_start + page_offset + ADDRESSES[:first_attr]
99
+
100
+ if virtual_page_number == ROOT_PAGE_NUMBER
101
+ # skipping root page analysis until it is further understood
102
+ is_root = true
103
+
104
+ elsif virtual_page_number == OBJECT_TABLE_PAGE_NUMBER
105
+ attributes = process_object_table_attributes(image, attributes_start)
106
+
107
+ else
108
+ attributes = process_attributes image, attributes_start
109
+ end
110
+
111
+ data[:pages][page][:sequence] = page_sequence
112
+ data[:pages][page][:virtual_page_number] = virtual_page_number
113
+
114
+ unless is_root
115
+ data[:pages][page][:attributes] = attributes
116
+ data[:pages][page].merge! inspect_attributes(attributes)
117
+ end
118
+ }
119
+ end
120
+
121
+ def volume_metadata(data, opts)
122
+ image = data[:image]
123
+ image_start = opts[:offset]
124
+
125
+ image.seek(image_start + ADDRESSES[:bytes_per_sector])
126
+ bytes_per_sector = image.read(4).unpack('L').first
127
+
128
+ image.seek(image_start + ADDRESSES[:sectors_per_cluster])
129
+ sectors_per_cluster = image.read(4).unpack('L').first
130
+
131
+ cluster_size = bytes_per_sector * sectors_per_cluster
132
+
133
+ {:bytes_per_sector => bytes_per_sector,
134
+ :sectors_per_cluster => sectors_per_cluster,
135
+ :cluster_size => cluster_size }
136
+ end
137
+
138
+ def pages(data, opts)
139
+ image = data[:image]
140
+ image_start = opts[:offset]
141
+ page = FIRST_PAGE_ID
142
+ pages = {}
143
+
144
+ image.seek(image_start + page * PAGE_SIZE)
145
+ while contents = image.read(PAGE_SIZE)
146
+ # only pull out metadata pages currently
147
+ is_metadata = contents.unpack('S').first == page
148
+ pages[page] = {:contents => contents} if is_metadata
149
+
150
+ page += 1
151
+ end
152
+
153
+ pages
154
+ end
155
+
156
+ # Convert an array of bytes in little endian order to human friendly string
157
+ def little_endian_str(bytes)
158
+ str = '0x'
159
+ value = false
160
+ bytes.reverse_each { |b|
161
+ next if b == 0 && !value
162
+ value = true
163
+ str += b.to_s(16)
164
+ }
165
+ str
166
+ end
167
+
168
+ def object_table_page_id(data)
169
+ # find shadow page w/ highest sequence
170
+ data[:pages].keys.select { |p| data[:pages][p][:virtual_page_number] == OBJECT_TABLE_PAGE_NUMBER }
171
+ .sort { |p1, p2| data[:pages][p2][:sequence] <=> data[:pages][p1][:sequence] }.first
172
+ end
173
+
174
+ def object_table(data, opts)
175
+ table = {}
176
+ page = data[:pages][object_table_page_id(data)]
177
+
178
+ # XXX this could start from the 2nd attribute if the exception in
179
+ # process_table_attributes does _not_ apply, need to investigate furthur / fix
180
+ page[:attributes][3...-1].each { |bytes|
181
+ # bytes 4-7 give us the key offset & length and
182
+ key_offset = bytes[4..5].pack('C*').unpack('S').first.to_i
183
+ key_length = bytes[6..7].pack('C*').unpack('S').first.to_i
184
+
185
+ # bytes A-D give us the value offset & length
186
+ value_offset = bytes[0xA..0xB].pack('C*').unpack('S').first.to_i
187
+ value_length = bytes[0xC..0xD].pack('C*').unpack('S').first.to_i
188
+
189
+ key = bytes[key_offset...key_offset+key_length]
190
+ value = bytes[value_offset...value_offset+value_length]
191
+
192
+ cluster_bytes = value[0..7]
193
+ # TODO extract 'type' from value[3a..3d]a (?)
194
+
195
+ object_id = key.pack('C*')
196
+ cluster = cluster_bytes.pack('C*')
197
+
198
+ object_str = little_endian_str(key)
199
+ cluster_str = little_endian_str(cluster_bytes)
200
+
201
+ table[object_id] = {:object_str => object_str,
202
+ :cluster => cluster,
203
+ :cluster_str => cluster_str}
204
+ }
205
+
206
+ table
207
+ end
208
+
209
+ def object_tree_page_id(data)
210
+ # find shadow page w/ highest sequence
211
+ data[:pages].keys.select { |p| data[:pages][p][:virtual_page_number] == OBJECT_TREE_PAGE_NUMBER }
212
+ .sort { |p1, p2| data[:pages][p2][:sequence] <=> data[:pages][p1][:sequence] }.first
213
+ end
214
+
215
+ def object_tree(data, opts)
216
+ tree = {}
217
+ page = data[:pages][object_tree_page_id(data)]
218
+
219
+ page[:attributes][2...-1].each { |bytes|
220
+ obj1_bytes = bytes[0x10..0x1F]
221
+ obj2_bytes = bytes[0x20..0x2F]
222
+
223
+ obj1 = little_endian_str(obj1_bytes)
224
+ obj2 = little_endian_str(obj2_bytes)
225
+
226
+ tree[obj1] ||= []
227
+ tree[obj1] << obj2
228
+ }
229
+
230
+ tree
231
+ end
232
+
233
+ def data_str(data, str_opts = {})
234
+ places = str_opts[:places] || 1
235
+
236
+ return '0x'+ ('0' * places) if data.nil?
237
+ '0x'+data.to_s(16).rjust(places, '0').upcase
238
+ end
239
+
240
+ def print_results(data, opts)
241
+ out = "Analyzed ReFS filesystem on #{opts[:image].green.bold} "\
242
+ "starting at #{opts[:offset].to_s.green.bold}\n" \
243
+ "VBR: #{data_str(data[:bytes_per_sector]).to_s.yellow.bold} (bytes per sector) * " \
244
+ "#{data_str(data[:sectors_per_cluster]).to_s.yellow.bold} (sectors per cluster) = " \
245
+ "#{data_str(data[:cluster_size]).to_s.yellow.bold} (bytes per cluster)\n"
246
+
247
+ data[:pages].keys.each { |page_id|
248
+ page = data[:pages][page_id]
249
+
250
+ page_out = "Page #{data_str(page_id, :places => 4).blue.bold}: "\
251
+ "number #{data_str(page[:virtual_page_number], :places => 3).blue.bold} - " \
252
+ "sequence #{data_str(page[:sequence], :places => 2).blue.bold} - " \
253
+ "object id #{data_str(page[:object_id], :places => 2).blue.bold} - " \
254
+ "records #{data_str(page[:entries], :places => 2).blue.bold}\n"
255
+
256
+ if opts[:attributes] && page[:attributes]
257
+ page_out += " Attributes:\n"
258
+ page[:attributes].each { |attr_values|
259
+ attr_out = attr_values.collect { |a| a.to_s(16) }.join(' ')[0...10] +'...'
260
+ page_out += ' ' + attr_out + "\n"
261
+ }
262
+ end
263
+
264
+ out += page_out
265
+ }
266
+
267
+ if opts[:object_table]
268
+ out += "\nObject table:\n"
269
+ out += "Obj | Cluster\n"
270
+ out += "-------------\n"
271
+ data[:object_table].keys.each { |obj_id|
272
+ object_str = data[:object_table][obj_id][:object_str]
273
+ cluster = data[:object_table][obj_id][:cluster_str]
274
+ out += "#{object_str[0..4]} | #{cluster}\n"
275
+ }
276
+ end
277
+
278
+ if opts[:object_tree]
279
+ out += "\nObject tree:\n"
280
+ out += "-------------\n"
281
+ data[:object_tree].keys.each { |obj_id|
282
+ references = data[:object_tree][obj_id].collect { |obj| obj[0..4] }.join(', ')
283
+ out += "#{obj_id[0..4]} -> #{references}\n"
284
+ }
285
+ end
286
+
287
+ puts out
288
+ end
289
+
290
+ def main(opts = {})
291
+ image = File.open(opts[:image], 'rb')
292
+
293
+ data = {}
294
+ data[:image] = image
295
+
296
+ data.merge! volume_metadata(data, opts)
297
+ data.merge! :pages => pages(data, opts)
298
+
299
+ parse_pages data, opts
300
+
301
+ data.merge! :object_table => object_table(data, opts)
302
+ data.merge! :object_tree => object_tree(data, opts)
303
+
304
+ print_results data, opts
305
+ end
306
+
307
+ def parse_cli(cli)
308
+ opts = {}
309
+ parser = OptionParser.new do |popts|
310
+ popts.on("-h", "--help", "Print help message") do
311
+ puts parser
312
+ exit
313
+ end
314
+
315
+ popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
316
+ opts[:image] = path
317
+ end
318
+
319
+ popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
320
+ opts[:offset] = offset.to_i
321
+ end
322
+
323
+ popts.on("-a", "--attributes", "Include attribute analysis in output") do
324
+ opts[:attributes] = true
325
+ end
326
+
327
+ popts.on("--table", "Include object table analysis in output") do
328
+ opts[:object_table] = true
329
+ end
330
+
331
+ popts.on("--tree", "Include object tree analysis in output") do
332
+ opts[:object_tree] = true
333
+ end
334
+ end
335
+
336
+ begin
337
+ parser.parse!(cli)
338
+ rescue OptionParser::InvalidOption
339
+ puts parser
340
+ exit
341
+ end
342
+
343
+ if !opts[:image] || !opts[:offset]
344
+ puts "--image and --offset params are needed at a minimum"
345
+ exit 1
346
+ end
347
+
348
+ opts
349
+ end
350
+
351
+ main parse_cli(ARGV) if __FILE__ == $0
@@ -0,0 +1,74 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS File Extractor
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'optparse'
6
+ require 'resilience'
7
+
8
+ def main
9
+ cli_opts = parse_cli(ARGV)
10
+ results = parse_image cli_opts
11
+ write_results cli_opts, results
12
+ end
13
+
14
+ def write_results(opts, results)
15
+ dir = opts[:dir]
16
+ Dir.mkdir(dir) unless File.directory?(dir)
17
+
18
+ results[:files].each do |name, contents|
19
+ puts "Got #{name}"
20
+ path = "#{dir}/#{name}".delete("\0")
21
+ File.write(path, contents)
22
+ end
23
+ end
24
+
25
+ def parse_image(opts)
26
+ file = File.open(opts[:image], 'rb')
27
+ image = Resilience::OnImage.image
28
+ image.file = file
29
+ image.offset = opts[:offset]
30
+ image.opts = opts
31
+
32
+ image.parse
33
+ files = image.root_dir.files
34
+ dirs = image.root_dir.dirs
35
+ {:files => files, :dirs => dirs}
36
+ end
37
+
38
+ def parse_cli(cli)
39
+ opts = {}
40
+ parser = OptionParser.new do |popts|
41
+ popts.on("-h", "--help", "Print help message") do
42
+ puts parser
43
+ exit
44
+ end
45
+
46
+ popts.on("-i", "--image path", "Path to the disk image to parse") do |path|
47
+ opts[:image] = path
48
+ end
49
+
50
+ popts.on("-o", "--offset bytes", "Start of volume with ReFS filesystem") do |offset|
51
+ opts[:offset] = offset.to_i
52
+ end
53
+
54
+ popts.on("-d", "--dir dir", "Output directory") do |dir|
55
+ opts[:dir] = dir
56
+ end
57
+ end
58
+
59
+ begin
60
+ parser.parse!(cli)
61
+ rescue OptionParser::InvalidOption
62
+ puts parser
63
+ exit
64
+ end
65
+
66
+ if !opts[:image] || !opts[:offset] || !opts[:dir]
67
+ puts "--image, --offset, and --dir params are needed"
68
+ exit 1
69
+ end
70
+
71
+ opts
72
+ end
73
+
74
+ main if __FILE__ == $0
@@ -0,0 +1,13 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Parser (expiremental)
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'resilience/mixins'
6
+ require 'resilience/constants'
7
+
8
+ require 'resilience/fs_dir'
9
+ require 'resilience/attribute'
10
+ require 'resilience/image'
11
+
12
+ require 'resilience/dirs'
13
+ require 'resilience/tables'
@@ -0,0 +1,28 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Attributes
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ class Attribute
7
+ include OnImage
8
+
9
+ attr_accessor :bytes
10
+
11
+ def initialize(args={})
12
+ @bytes = args[:bytes] if args.key?(:bytes)
13
+ end
14
+
15
+ def self.read
16
+ pos = image.pos
17
+ attr_len = image.read(4).unpack('L').first
18
+ return new if attr_len == 0
19
+
20
+ image.seek pos
21
+ new(:bytes => image.read(attr_len))
22
+ end
23
+
24
+ def unpack(format)
25
+ bytes.unpack(format)
26
+ end
27
+ end
28
+ end # module Resilience
@@ -0,0 +1,32 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Constants
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ FIRST_PAGE_ID = 0x1e
6
+ PAGE_SIZE = 0x4000
7
+ FIRST_PAGE_ADDRESS = FIRST_PAGE_ID * PAGE_SIZE
8
+
9
+ ROOT_DIR_ID = [0, 0, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0]
10
+ DIR_ENTRY = 0x20030
11
+ FILE_ENTRY = 0x10030
12
+
13
+ DIR_TREE = 0x301
14
+ DIR_LIST = 0x200
15
+ #DIR_BRANCH = 0x000 ?
16
+
17
+ ADDRESSES = {
18
+ # page
19
+ :virtual_page_number => 0x18,
20
+ :first_attr => 0x30,
21
+
22
+ # page 0x1e
23
+ :system_table_page => 0xA0,
24
+
25
+ # system table
26
+ :system_pages => 0x58,
27
+
28
+ # generic table
29
+ :num_objects => 0x20, # referenced from start of first attr
30
+
31
+ :table_length => 0x04 # referenced from start of table header
32
+ }
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Directories
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'resilience/dirs/root_dir'
@@ -0,0 +1,15 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Root Dir
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ class RootDir < FSDir::DirBase
7
+ include OnImage
8
+
9
+ def self.parse
10
+ dir = new
11
+ dir.parse_dir_obj ROOT_DIR_ID, ''
12
+ dir
13
+ end
14
+ end # class RootDir
15
+ end # module Resilience
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS FileSystem Directory
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'resilience/fs_dir/dir_base'
6
+ require 'resilience/fs_dir/record'
@@ -0,0 +1,106 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Directory Handling
3
+ # Copyright (C) 2014-2015 Red Hat Inc.
4
+
5
+ require 'fileutils'
6
+
7
+ module Resilience
8
+ module FSDir
9
+ class DirBase
10
+ include OnImage
11
+
12
+ attr_accessor :dirs
13
+ attr_accessor :files
14
+
15
+ def parse_dir_obj(object_id, prefix)
16
+ object_table = image.object_table
17
+ @dirs ||= {}
18
+ @files ||= {}
19
+
20
+ page_id = object_table.pages[object_id]
21
+ page_address = page_id * PAGE_SIZE
22
+ parse_dir_page page_address, prefix
23
+ end
24
+
25
+ def parse_dir_page(page_address, prefix)
26
+ # skip container/placeholder attribute
27
+ image.seek(page_address + ADDRESSES[:first_attr])
28
+ Attribute.read
29
+
30
+ # start of table attr, pull out table length, type
31
+ table_header_attr = Attribute.read
32
+ table_header_dwords = table_header_attr.unpack("L*")
33
+ header_len = table_header_dwords[0]
34
+ table_len = table_header_dwords[1]
35
+ remaining_len = table_len - header_len
36
+ table_type = table_header_dwords[3]
37
+
38
+ until remaining_len == 0
39
+ orig_pos = image.pos
40
+ record = Record.read
41
+
42
+ # need to keep track of position locally as we
43
+ # recursively call parse_dir via helpers
44
+ pos = image.pos
45
+
46
+ if table_type == DIR_TREE
47
+ parse_dir_branch record, prefix
48
+
49
+ else #if table_type == DIR_LIST
50
+ record = filter_dir_record(record)
51
+ pos = image.pos
52
+ parse_dir_record record, prefix
53
+
54
+ end
55
+
56
+ image.seek pos
57
+ remaining_len -= (image.pos - orig_pos)
58
+ end
59
+ end
60
+
61
+ def filter_dir_record(record)
62
+ # '4' seems to indicate a historical record or similar,
63
+ # records w/ flags '0' or '8' are what we want
64
+ record.flags == 4 ? filter_dir_record(Record.read) : record
65
+ end
66
+
67
+ def parse_dir_branch(record, prefix)
68
+ key = record.key
69
+ value = record.value
70
+ flags = record.flags
71
+
72
+ value_dwords = value.unpack('L*')
73
+ value_qwords = value.unpack('Q*')
74
+
75
+ page_id = value_dwords[0]
76
+ page_address = page_id * PAGE_SIZE
77
+ checksum = value_qwords[2]
78
+
79
+ parse_dir_page page_address, prefix unless checksum == 0 || flags == 4
80
+ end
81
+
82
+ def parse_dir_record(record, prefix)
83
+ key = record.key
84
+ value = record.value
85
+
86
+ key_bytes = key.unpack('C*')
87
+ key_dwords = key.unpack('L*')
88
+ entry_type = key_dwords.first
89
+
90
+ if entry_type == DIR_ENTRY
91
+ dir_name = key_bytes[4..-1].pack('L*')
92
+ dir_obj = value.unpack('C*')[0...8]
93
+ dirs["#{prefix}\\#{dir_name}"] = dir_obj
94
+
95
+ dir_obj = [0, 0, 0, 0, 0, 0, 0, 0].concat(dir_obj)
96
+ parse_dir_obj(dir_obj, "#{prefix}\\#{dir_name}")
97
+
98
+ elsif entry_type == FILE_ENTRY
99
+ file_name = key_bytes[4..-1].pack('L*')
100
+ prefixed = "#{prefix}\\#{file_name}"
101
+ files[prefixed] = value
102
+ end
103
+ end
104
+ end # class DirBase
105
+ end # module FSDir
106
+ end # module Resilience
@@ -0,0 +1,59 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Records
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ module FSDir
7
+ class Record
8
+ include OnImage
9
+
10
+ attr_accessor :attribute
11
+
12
+ def initialize(attribute)
13
+ @attribute = attribute
14
+ end
15
+
16
+ def self.read
17
+ new(Attribute.read)
18
+ end
19
+
20
+ def calc_boundries
21
+ return if @boundries_set
22
+ @boundries_set = true
23
+
24
+ header = attribute.unpack('S*')
25
+ @key_offset = header[2]
26
+ @key_length = header[3]
27
+ @value_offset = header[5]
28
+ @value_length = header[6]
29
+ end
30
+
31
+ def boundries
32
+ calc_boundries
33
+ [@key_offset, @key_length, @value_offset, @value_length]
34
+ end
35
+
36
+ def calc_flags
37
+ return if @flags_set
38
+ @flags_set = true
39
+
40
+ @flags = attribute.unpack('S*')[4]
41
+ end
42
+
43
+ def flags
44
+ calc_flags
45
+ @flags
46
+ end
47
+
48
+ def key
49
+ ko, kl, vo, vl = boundries
50
+ attribute.unpack('C*')[ko...ko+kl].pack('C*')
51
+ end
52
+
53
+ def value
54
+ ko, kl, vo, vl = boundries
55
+ attribute.unpack('C*')[vo..-1].pack('C*')
56
+ end
57
+ end # class Record
58
+ end # module FSDir
59
+ end # module Resilience
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Image Representation
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ class Image
7
+ attr_accessor :file
8
+ attr_accessor :offset
9
+ attr_accessor :opts
10
+
11
+ attr_accessor :root_dir
12
+ attr_accessor :system_table
13
+ attr_accessor :object_table
14
+
15
+ def initialize(args={})
16
+ @file = args[:file] || []
17
+ end
18
+
19
+ def parse
20
+ @system_table = Resilience::SystemTable.parse
21
+ @object_table = Resilience::ObjectTable.parse
22
+ @root_dir = Resilience::RootDir.parse
23
+ end
24
+
25
+ def seek(position)
26
+ @file.seek offset + position
27
+ end
28
+
29
+ def pos
30
+ @file.pos - offset
31
+ end
32
+
33
+ def read(len)
34
+ @file.read(len)
35
+ end
36
+ end
37
+ end # module Resilience
@@ -0,0 +1,5 @@
1
+ #!/usr/bin/ruby
2
+ # Resilience Mixins
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'resilience/mixins/on_image'
@@ -0,0 +1,25 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS On Image Mixin
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ module OnImage
7
+ def self.included(base)
8
+ base.extend(ClassMethods)
9
+ end
10
+
11
+ def self.image
12
+ @image ||= Resilience::Image.new
13
+ end
14
+
15
+ def image
16
+ OnImage.image
17
+ end
18
+
19
+ module ClassMethods
20
+ def image
21
+ OnImage.image
22
+ end
23
+ end
24
+ end # module OnImage
25
+ end # module Resilience
@@ -0,0 +1,6 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Tables
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ require 'resilience/tables/object'
6
+ require 'resilience/tables/system'
@@ -0,0 +1,46 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS Object Table
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ class ObjectTable
7
+ include OnImage
8
+
9
+ attr_accessor :pages
10
+
11
+ def initialize
12
+ @pages ||= {}
13
+ end
14
+
15
+ def self.parse
16
+ table = new
17
+ table.parse_pages
18
+ table
19
+ end
20
+
21
+ def parse_pages
22
+ # in the images I've seen this has always been the first entry
23
+ # in the system table, though always has virtual page id = 2
24
+ # which we could look for if this turns out not to be the case
25
+ object_page_id = image.system_table.pages.first
26
+ object_page_address = object_page_id * PAGE_SIZE
27
+
28
+ # read number of objects from index header
29
+ image.seek(object_page_address + ADDRESSES[:first_attr])
30
+ first_attr = Attribute.read
31
+ num_objects = first_attr.unpack('L*')[ADDRESSES[:num_objects]/4]
32
+
33
+ # start of table attr, skip for now
34
+ Attribute.read
35
+
36
+ 0.upto(num_objects-1) do
37
+ object_record = FSDir::Record.read
38
+ object_id = object_record.key.unpack('C*')
39
+
40
+ # here object page is first qword of record value
41
+ object_page = object_record.value.unpack('Q*').first
42
+ @pages[object_id] = object_page
43
+ end
44
+ end
45
+ end # class ObjectTable
46
+ end # module Resilience
@@ -0,0 +1,42 @@
1
+ #!/usr/bin/ruby
2
+ # ReFS System Table
3
+ # Copyright (C) 2015 Red Hat Inc.
4
+
5
+ module Resilience
6
+ class SystemTable
7
+ include OnImage
8
+
9
+ attr_accessor :pages
10
+
11
+ def initialize
12
+ @pages = []
13
+ end
14
+
15
+ def self.parse
16
+ table = new
17
+ table.parse_pages
18
+ table
19
+ end
20
+
21
+ def parse_pages
22
+ image.seek(FIRST_PAGE_ADDRESS + ADDRESSES[:system_table_page])
23
+ system_table_page = image.read(8).unpack('Q').first
24
+ system_table_address = system_table_page * PAGE_SIZE
25
+
26
+ image.seek(system_table_address + ADDRESSES[:system_pages])
27
+ num_system_pages = image.read(4).unpack('L').first
28
+
29
+ 0.upto(num_system_pages-1) do
30
+ system_page_offset = image.read(4).unpack('L').first
31
+ pos = image.pos
32
+
33
+ image.seek(system_table_address + system_page_offset)
34
+ system_page = image.read(8).unpack('Q').first
35
+ @pages << system_page
36
+
37
+ image.seek(pos)
38
+ end
39
+ end
40
+ end # class SystemTable
41
+ end # module Resilience
42
+
metadata ADDED
@@ -0,0 +1,72 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: resilience
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.0.2
5
+ platform: ruby
6
+ authors:
7
+ - Mo Morsi
8
+ autorequire:
9
+ bindir: bin
10
+ cert_chain: []
11
+ date: 2015-05-23 00:00:00.000000000 Z
12
+ dependencies: []
13
+ description: Ruby ReFS utils
14
+ email: mmorsi@redhat.com
15
+ executables:
16
+ - fs.rb
17
+ - resilience.rb
18
+ - rex.rb
19
+ - rels.rb
20
+ - ref.rb
21
+ - clum.py
22
+ - cle.rb
23
+ extensions: []
24
+ extra_rdoc_files: []
25
+ files:
26
+ - README.md
27
+ - bin/cle.rb
28
+ - bin/clum.py
29
+ - bin/fs.rb
30
+ - bin/ref.rb
31
+ - bin/rels.rb
32
+ - bin/resilience.rb
33
+ - bin/rex.rb
34
+ - lib/resilience.rb
35
+ - lib/resilience/attribute.rb
36
+ - lib/resilience/constants.rb
37
+ - lib/resilience/dirs.rb
38
+ - lib/resilience/dirs/root_dir.rb
39
+ - lib/resilience/fs_dir.rb
40
+ - lib/resilience/fs_dir/dir_base.rb
41
+ - lib/resilience/fs_dir/record.rb
42
+ - lib/resilience/image.rb
43
+ - lib/resilience/mixins.rb
44
+ - lib/resilience/mixins/on_image.rb
45
+ - lib/resilience/tables.rb
46
+ - lib/resilience/tables/object.rb
47
+ - lib/resilience/tables/system.rb
48
+ homepage: https://gist.github.com/movitto/866de4356f56a3b478ca
49
+ licenses:
50
+ - MIT
51
+ metadata: {}
52
+ post_install_message:
53
+ rdoc_options: []
54
+ require_paths:
55
+ - lib
56
+ required_ruby_version: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - ">="
59
+ - !ruby/object:Gem::Version
60
+ version: '0'
61
+ required_rubygems_version: !ruby/object:Gem::Requirement
62
+ requirements:
63
+ - - ">="
64
+ - !ruby/object:Gem::Version
65
+ version: '0'
66
+ requirements: []
67
+ rubyforge_project:
68
+ rubygems_version: 2.2.2
69
+ signing_key:
70
+ specification_version: 4
71
+ summary: A module/command-line utility to parse a ReFS file system image
72
+ test_files: []