pedump 0.5.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/Gemfile +16 -0
- data/Gemfile.lock +90 -0
- data/LICENSE.txt +20 -0
- data/README.md +410 -0
- data/Rakefile +179 -0
- data/VERSION +1 -0
- data/bin/pedump +7 -0
- data/data/fs.txt +224 -0
- data/data/jc-userdb.txt +14371 -0
- data/data/sig.bin +0 -0
- data/data/signatures.txt +678 -0
- data/data/userdb.txt +14083 -0
- data/lib/pedump.rb +868 -0
- data/lib/pedump/cli.rb +804 -0
- data/lib/pedump/comparer.rb +147 -0
- data/lib/pedump/composite_io.rb +56 -0
- data/lib/pedump/core.rb +38 -0
- data/lib/pedump/core_ext/try.rb +57 -0
- data/lib/pedump/loader.rb +393 -0
- data/lib/pedump/loader/minidump.rb +351 -0
- data/lib/pedump/loader/section.rb +57 -0
- data/lib/pedump/logger.rb +67 -0
- data/lib/pedump/ne.rb +425 -0
- data/lib/pedump/ne/version_info.rb +171 -0
- data/lib/pedump/packer.rb +173 -0
- data/lib/pedump/pe.rb +121 -0
- data/lib/pedump/resources.rb +436 -0
- data/lib/pedump/security.rb +58 -0
- data/lib/pedump/sig_parser.rb +507 -0
- data/lib/pedump/tls.rb +17 -0
- data/lib/pedump/unpacker.rb +26 -0
- data/lib/pedump/unpacker/aspack.rb +858 -0
- data/lib/pedump/unpacker/upx.rb +13 -0
- data/lib/pedump/version.rb +10 -0
- data/lib/pedump/version_info.rb +171 -0
- data/misc/aspack/Makefile +3 -0
- data/misc/aspack/aspack_unlzx.c +92 -0
- data/misc/aspack/lzxdec.c +479 -0
- data/misc/aspack/lzxdec.h +56 -0
- data/misc/nedump.c +751 -0
- data/pedump.gemspec +109 -0
- metadata +227 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
class PEdump
|
2
|
+
def security f=@io
|
3
|
+
return nil unless pe(f) && pe(f).ioh && f
|
4
|
+
dir = @pe.ioh.DataDirectory[IMAGE_DATA_DIRECTORY::SECURITY]
|
5
|
+
return nil if !dir || dir.va == 0
|
6
|
+
|
7
|
+
# IMAGE_DIRECTORY_ENTRY_SECURITY
|
8
|
+
# Points to a list of WIN_CERTIFICATE structures, defined in WinTrust.H.
|
9
|
+
# Not mapped into memory as part of the image.
|
10
|
+
# Therefore, the VirtualAddress field is a file offset, rather than an RVA.
|
11
|
+
#
|
12
|
+
# http://msdn.microsoft.com/en-us/magazine/bb985997.aspx
|
13
|
+
|
14
|
+
f.seek dir.va
|
15
|
+
r = []
|
16
|
+
ofs = f.tell
|
17
|
+
while !f.eof? && (f.tell-ofs < dir.size)
|
18
|
+
r << WIN_CERTIFICATE.read(f)
|
19
|
+
end
|
20
|
+
r
|
21
|
+
end
|
22
|
+
alias :signature :security
|
23
|
+
|
24
|
+
# WIN_CERT_TYPE_X509 (0x0001) bCertificate contains an X.509 certificate.
|
25
|
+
# WIN_CERT_TYPE_PKCS_SIGNED_DATA (0x0002) bCertificate contains a PKCS SignedData structure.
|
26
|
+
# WIN_CERT_TYPE_RESERVED_1 (0x0003) Reserved.
|
27
|
+
# WIN_CERT_TYPE_PKCS1_SIGN (0x0009) bCertificate contains PKCS1_MODULE_SIGN fields.
|
28
|
+
|
29
|
+
# http://msdn.microsoft.com/en-us/library/aa447037.aspx
|
30
|
+
class WIN_CERTIFICATE < IOStruct.new 'Vvv',
|
31
|
+
:dwLength,
|
32
|
+
:wRevision,
|
33
|
+
:wCertificateType,
|
34
|
+
# manual
|
35
|
+
:data
|
36
|
+
|
37
|
+
def self.read f
|
38
|
+
super.tap do |x|
|
39
|
+
if x.dwLength.to_i < 8
|
40
|
+
PEdump.logger.error "[!] #{x.class}: too small length #{x.dwLength}"
|
41
|
+
elsif x.dwLength.to_i > 0x100_000
|
42
|
+
PEdump.logger.error "[!] #{x.class}: too big length #{x.dwLength}"
|
43
|
+
else
|
44
|
+
x.data = f.read(x.dwLength - 8)
|
45
|
+
begin
|
46
|
+
case x.wCertificateType
|
47
|
+
when 2
|
48
|
+
require 'openssl'
|
49
|
+
x.data = OpenSSL::PKCS7.new(x.data)
|
50
|
+
end
|
51
|
+
rescue
|
52
|
+
PEdump.logger.error "[!] #{$!}"
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,507 @@
|
|
1
|
+
class PEdump
|
2
|
+
module SigParser
|
3
|
+
|
4
|
+
DATA_ROOT = File.dirname(File.dirname(File.dirname(__FILE__)))
|
5
|
+
|
6
|
+
TEXT_SIGS_FILES = [
|
7
|
+
File.join(DATA_ROOT, "data", "userdb.txt"),
|
8
|
+
File.join(DATA_ROOT, "data", "signatures.txt"),
|
9
|
+
File.join(DATA_ROOT, "data", "jc-userdb.txt"),
|
10
|
+
File.join(DATA_ROOT, "data", "fs.txt"), # has special parse options!
|
11
|
+
]
|
12
|
+
|
13
|
+
SPECIAL_PARSE_OPTIONS = {
|
14
|
+
File.join(DATA_ROOT, "data", "fs.txt") => {:fix1 => true}
|
15
|
+
}
|
16
|
+
|
17
|
+
class OrBlock < Array; end
|
18
|
+
|
19
|
+
class << self
|
20
|
+
|
21
|
+
# parse text signatures
|
22
|
+
def parse args = {}
|
23
|
+
args[:fnames] ||= TEXT_SIGS_FILES
|
24
|
+
sigs = {}; sig = nil
|
25
|
+
|
26
|
+
args[:fnames].each do |fname|
|
27
|
+
n0 = sigs.size; add_sig_args = args.dup
|
28
|
+
add_sig_args.merge!(SPECIAL_PARSE_OPTIONS[fname] || {})
|
29
|
+
File.open(fname,'r:utf-8') do |f|
|
30
|
+
while line = f.gets
|
31
|
+
case line.strip
|
32
|
+
when /^[<;#]/, /^$/ # comments & blank lines
|
33
|
+
next
|
34
|
+
when /^\[(.+)=(.+)\]$/
|
35
|
+
_add_sig(sigs, Packer.new($1, $2, true), add_sig_args )
|
36
|
+
when /^\[([^=]+)\](\s+\/\/.+)?$/
|
37
|
+
sig = Packer.new($1)
|
38
|
+
when /^signature = (.+)$/
|
39
|
+
sig.re = $1
|
40
|
+
_add_sig(sigs, sig, add_sig_args)
|
41
|
+
when /^ep_only = (.+)$/
|
42
|
+
sig.ep_only = ($1.strip.downcase == 'true')
|
43
|
+
else raise line
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
puts "[=] #{sigs.size-n0} sigs from #{File.basename(fname)}\n\n" if args[:verbose]
|
48
|
+
end
|
49
|
+
|
50
|
+
bins = Hash.new{ |k,v| k[v] = ''.force_encoding('binary') }
|
51
|
+
|
52
|
+
# convert strings to Regexps
|
53
|
+
sigs = sigs.values
|
54
|
+
sigs.each_with_index do |sig,idx|
|
55
|
+
sig.re =
|
56
|
+
sig.re.split(' ').tap do |a|
|
57
|
+
sig.size = a.size
|
58
|
+
end.map do |x|
|
59
|
+
case x
|
60
|
+
when /\A\?\?\Z/
|
61
|
+
bins[sig] << '.'
|
62
|
+
'.'
|
63
|
+
when /\A.\?/,/\?.\Z/
|
64
|
+
puts "[?] #{x.inspect} -> \"??\" in #{sig.name}" if args[:verbose]
|
65
|
+
bins[sig] << '.'
|
66
|
+
'.'
|
67
|
+
when /\A[a-f0-9]{2}\Z/i
|
68
|
+
x = x.to_i(16).chr
|
69
|
+
bins[sig] << x
|
70
|
+
if args[:raw]
|
71
|
+
x
|
72
|
+
elsif args[:raword]
|
73
|
+
x.ord
|
74
|
+
else
|
75
|
+
Regexp::escape(x)
|
76
|
+
end
|
77
|
+
else
|
78
|
+
puts "[?] unknown re element: #{x.inspect} in #{sig.inspect}" if args[:verbose]
|
79
|
+
"BAD_RE"
|
80
|
+
break
|
81
|
+
end
|
82
|
+
end
|
83
|
+
if sig.name[/-+>/]
|
84
|
+
a = sig.name.split(/-+>/,2).map(&:strip)
|
85
|
+
sig.name = "#{a[0]} (#{a[1]})"
|
86
|
+
end
|
87
|
+
sig.re.pop while sig.re && sig.re.last == '??'
|
88
|
+
end
|
89
|
+
sigs.delete_if{ |sig| !sig.re || sig.re.index('BAD_RE') }
|
90
|
+
return sigs if args[:raw] || args[:raword]
|
91
|
+
|
92
|
+
# require 'awesome_print'
|
93
|
+
# bins.each do |bin_sig, bin|
|
94
|
+
# next if bin.size < 5
|
95
|
+
# #next unless bin_sig.name['UPX']
|
96
|
+
#
|
97
|
+
# bin_re = Regexp.new(bin_sig.re.join, Regexp::MULTILINE)
|
98
|
+
# was = false
|
99
|
+
# sigs.each do |sig|
|
100
|
+
# next if sig.size < 5 || sig == bin_sig
|
101
|
+
# #next unless sig.name['UPX']
|
102
|
+
#
|
103
|
+
# re = Regexp.new(sig.re.join, Regexp::MULTILINE)
|
104
|
+
# if bin.index(re) == 0
|
105
|
+
# rd = _re_diff(bin_re.source, re.source)
|
106
|
+
# if rd.any? && rd.size <= 4
|
107
|
+
# #if sig.name.split.first.upcase != bin_sig.name.split.first.upcase
|
108
|
+
# puts "\n[.] #{bin_sig.name.yellow}\n#{bin_re.source.inspect.red}" unless was
|
109
|
+
# puts "[=] #{sig.name}"
|
110
|
+
# puts re.source.inspect.green
|
111
|
+
# p rd
|
112
|
+
# was = true
|
113
|
+
# #end
|
114
|
+
# end
|
115
|
+
# end
|
116
|
+
# end
|
117
|
+
# end
|
118
|
+
|
119
|
+
|
120
|
+
optimize sigs if args[:optimize]
|
121
|
+
|
122
|
+
# convert re-arrays to Regexps
|
123
|
+
sigs.each do |sig|
|
124
|
+
sig.re = Regexp.new( _join(sig.re), Regexp::MULTILINE )
|
125
|
+
end
|
126
|
+
|
127
|
+
sigs
|
128
|
+
end
|
129
|
+
|
130
|
+
# XXX
|
131
|
+
# "B\xE9rczi G\xE1bor".force_encoding('binary').to_yaml:
|
132
|
+
# RuntimeError: expected SCALAR, SEQUENCE-START, MAPPING-START, or ALIAS
|
133
|
+
|
134
|
+
def _add_sig sigs, sig, args = {}
|
135
|
+
raise "null RE: #{sig.inspect}" unless sig.re
|
136
|
+
|
137
|
+
# bad sigs
|
138
|
+
return if sig.re[/\A538BD833C0A30:::::/]
|
139
|
+
return if sig.name == "Name of the Packer v1.0"
|
140
|
+
return if sig.name == "Alias PIX/Vivid IMG Graphics format"
|
141
|
+
return if sig.name == "JAR Archive"
|
142
|
+
return if sig.name == "Turbo / Borland Pascal v7.x Unit"
|
143
|
+
return if sig.re == "54 68 69 73 20 70 72 6F 67 72 61 6D 20 63 61 6E 6E 6F 74 20 62 65 20 72 75 6E 20 69 6E 20 44 4F 53 20 6D 6F" # dos stub
|
144
|
+
|
145
|
+
sig.name.sub!(/^\*\s+/, '')
|
146
|
+
sig.name.sub!(/\s+\(h\)$/, '')
|
147
|
+
sig.name.sub!(/version (\d)/i,"v\\1")
|
148
|
+
sig.name.sub!(/Microsoft/i, "MS")
|
149
|
+
sig.name.sub!(/ or /i, " / ")
|
150
|
+
sig.name.sub! 'RLP ','RLPack '
|
151
|
+
sig.name.sub! '.beta', ' beta'
|
152
|
+
sig.name.sub! '(com)','[com]'
|
153
|
+
sig.name.gsub!(/ V(\d)/, " v\\1") # V1.1 -> v1.1
|
154
|
+
sig.name = sig.name.split(/\s*-+>\s*/).join(' -> ') # fix spaces around '->'
|
155
|
+
sig.name = sig.name.split(' ').delete_if do |x|
|
156
|
+
# delete words: vX.X, v?.?, ?.?, x.x
|
157
|
+
x =~ /\Av?[?x]\.[?x]\Z/i
|
158
|
+
end.join(' ')
|
159
|
+
|
160
|
+
sig.re = sig.re.strip.upcase.tr(':','?')
|
161
|
+
sig.re = sig.re.scan(/../).join(' ') if sig.re.split.first.size > 2
|
162
|
+
|
163
|
+
# sig contains entirely zeroes or masks or only both
|
164
|
+
a_bad = [%w'00', %w'??', %w'00 ??', %w'90', %w'90 ??']
|
165
|
+
# ?a, 0? => ??, ??
|
166
|
+
a_cur = sig.re.split.map{ |x| x['?'] ? '??' : x }.uniq.sort
|
167
|
+
return if a_bad.include?(a_cur)
|
168
|
+
|
169
|
+
# first byte is unique and all others are zeroes or masks
|
170
|
+
a_cur = sig.re.split[1..-1].map{ |x| x['?'] ? '??' : x }.uniq.sort
|
171
|
+
return if a_bad.include?(a_cur)
|
172
|
+
|
173
|
+
# too short signatures
|
174
|
+
if sig.re.split.delete_if{ |x| x['?'] }.size < 3
|
175
|
+
require 'awesome_print'
|
176
|
+
puts sig.inspect.red
|
177
|
+
end
|
178
|
+
|
179
|
+
# fs.txt contains a lot of signatures that copied from other sources
|
180
|
+
# BUT have all 01 replaced with '??'
|
181
|
+
# // replaced the file with filtered one (see 'fs-good' below) // zzz
|
182
|
+
if args[:fix1]
|
183
|
+
sigs.keys.each do |re|
|
184
|
+
if re.gsub("01","??") == sig.re
|
185
|
+
puts "[.] fix1: rejecting #{sig.name} - already present with 01 in place" if args[:verbose]
|
186
|
+
return
|
187
|
+
end
|
188
|
+
end
|
189
|
+
# File.open("fs-good.txt","a") do |f|
|
190
|
+
# f << "[#{sig.name}=#{sig.re.tr(' ','')}]\n"
|
191
|
+
# end
|
192
|
+
end
|
193
|
+
|
194
|
+
if sigs[sig.re]
|
195
|
+
a = [sig, sigs[sig.re]].map{ |x| x.name.upcase.split('->').first.tr('V ','') }
|
196
|
+
return if a[0][a[1]] || a[1][a[0]]
|
197
|
+
|
198
|
+
new_name = _merge_names(sigs[sig.re].name, sig.name)
|
199
|
+
if new_name && new_name != sig.name && new_name != sigs[sig.re].name
|
200
|
+
puts "[.] sig name join: #{new_name}" if args[:verbose]
|
201
|
+
sigs[sig.re].name = new_name
|
202
|
+
end
|
203
|
+
else
|
204
|
+
# new sig
|
205
|
+
sigs[sig.re] = sig
|
206
|
+
end
|
207
|
+
end
|
208
|
+
|
209
|
+
def _merge_names name1, name2
|
210
|
+
a = [name1, name2].map{ |x| x.split('->').first.split }
|
211
|
+
|
212
|
+
d = [a[0]-a[1], a[1]-a[0]] # different words
|
213
|
+
d.map! do |x|
|
214
|
+
x - [
|
215
|
+
'EXE','[EXE]',
|
216
|
+
'DLL','(DLL)','[DLL]',
|
217
|
+
'[LZMA]','(LZMA)','LZMA',
|
218
|
+
'-','~','(pack)','(1)','(2)',
|
219
|
+
'19??',
|
220
|
+
'with:', 'with?'
|
221
|
+
]
|
222
|
+
end
|
223
|
+
return if d.all?(&:empty?) # no different words => can keep ANY name
|
224
|
+
|
225
|
+
|
226
|
+
# if name1 =~ /pecompact/i
|
227
|
+
# require 'awesome_print'
|
228
|
+
# puts "[d] #{name1}".yellow
|
229
|
+
# puts "[d] #{name2}".yellow
|
230
|
+
# end
|
231
|
+
|
232
|
+
# [["v1.14/v1.20"], ["v1.14,", "v1.20"]]]
|
233
|
+
# [["EXEShield", "v0.3b/v0.3", "v0.6"], ["Shield", "v0.3b,", "v0.3"]]]
|
234
|
+
2.times do |i|
|
235
|
+
return if d[i].all? do |x|
|
236
|
+
x = x.downcase.delete(',-').sub(/tm$/,'')
|
237
|
+
d[1-i].any? do |y|
|
238
|
+
y = y.downcase.delete(',-').sub(/tm$/,'')
|
239
|
+
y[x]
|
240
|
+
end
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
# require 'awesome_print'
|
245
|
+
# puts "[d] #{name1.yellow} #{name2.green}"
|
246
|
+
|
247
|
+
a = name1.split
|
248
|
+
b = name2.split
|
249
|
+
|
250
|
+
# merge common head
|
251
|
+
new_name_head = []
|
252
|
+
while a.any? && b.any? && a.first.upcase == b.first.upcase
|
253
|
+
new_name_head << a.shift
|
254
|
+
b.shift
|
255
|
+
end
|
256
|
+
|
257
|
+
# merge common tail
|
258
|
+
new_name_tail = []
|
259
|
+
while a.any? && b.any? && a.last.upcase == b.last.upcase
|
260
|
+
new_name_tail.unshift a.pop
|
261
|
+
b.pop
|
262
|
+
end
|
263
|
+
|
264
|
+
# rm common words from middle
|
265
|
+
separators = [ "/", "->" ]
|
266
|
+
was = true
|
267
|
+
while was
|
268
|
+
was = false
|
269
|
+
b.each do |bw|
|
270
|
+
next if bw == "/" || bw == "->"
|
271
|
+
if a.include?(bw) || a.include?(bw+")") || a.include?("("+bw) || a.include?("(#{bw})")
|
272
|
+
b -= [bw]
|
273
|
+
was = true
|
274
|
+
break
|
275
|
+
end
|
276
|
+
end
|
277
|
+
end
|
278
|
+
while separators.include?(b.last)
|
279
|
+
b.pop
|
280
|
+
end
|
281
|
+
|
282
|
+
new_name = new_name_head
|
283
|
+
new_name << [a.join(' '), b.join(' ')].delete_if{|x| x.empty?}.join(' / ')
|
284
|
+
new_name += new_name_tail
|
285
|
+
# if name1 =~ /pecompact/i
|
286
|
+
# p a
|
287
|
+
# p b
|
288
|
+
# p new_name_tail
|
289
|
+
# puts "[=] #{new_name.inspect}".red
|
290
|
+
# end
|
291
|
+
new_name = new_name.join(' ')
|
292
|
+
end
|
293
|
+
|
294
|
+
def _join a, sep=''
|
295
|
+
a.map do |x|
|
296
|
+
case x
|
297
|
+
when OrBlock
|
298
|
+
'(' + _join(x, '|') + ')'
|
299
|
+
when Array
|
300
|
+
_join x
|
301
|
+
when String
|
302
|
+
x
|
303
|
+
end
|
304
|
+
end.join(sep)
|
305
|
+
end
|
306
|
+
|
307
|
+
def _re_diff a,b, max_cnt = 1000
|
308
|
+
r = []
|
309
|
+
[a,b].map(&:size).max.times.map do |i|
|
310
|
+
if a[i] != b[i]
|
311
|
+
r << [a[i],b[i]]
|
312
|
+
return nil if r.size > max_cnt
|
313
|
+
end
|
314
|
+
end
|
315
|
+
r
|
316
|
+
end
|
317
|
+
|
318
|
+
def _optimize sigs
|
319
|
+
nfound = 0
|
320
|
+
min_sz = 6
|
321
|
+
max_diff = 6
|
322
|
+
sigs.each_with_index do |sig1,idx|
|
323
|
+
#break if idx == 100
|
324
|
+
next if sig1.re.size < min_sz
|
325
|
+
next if sig1.name['PseudoSigner']
|
326
|
+
|
327
|
+
sigs[(idx+1)..-1].each do |sig2|
|
328
|
+
next if sig2.re.size < min_sz
|
329
|
+
next if sig2.name['PseudoSigner']
|
330
|
+
|
331
|
+
if rd = _re_diff(sig1.re, sig2.re, max_diff)
|
332
|
+
if rd.all?{ |x| x[0].nil? || x[0] == '.' } && sig2.re.size >= sig1.re.size
|
333
|
+
if new_name = _merge_names(sig2.name, sig1.name)
|
334
|
+
# require 'pp'
|
335
|
+
# pp ["FIRST", sig1.name, sig2.name, new_name, sig1.re.join, sig2.re.join] if new_name =~ /pecompact/i
|
336
|
+
sig1.name = new_name
|
337
|
+
end
|
338
|
+
sig2.ep_only ||= sig1.ep_only
|
339
|
+
sig2.re = []
|
340
|
+
elsif rd.all?{ |x| x[1].nil? || x[1] == '.' } && sig1.re.size >= sig2.re.size
|
341
|
+
if new_name = _merge_names(sig2.name, sig1.name)
|
342
|
+
# require 'pp'
|
343
|
+
# pp ["SECOND", sig1.name, sig2.name, new_name, sig1.re.join, sig2.re.join] if new_name =~ /pecompact/i
|
344
|
+
sig2.name = new_name
|
345
|
+
end
|
346
|
+
sig1.re = []
|
347
|
+
sig1.ep_only ||= sig2.ep_only
|
348
|
+
break
|
349
|
+
else
|
350
|
+
next
|
351
|
+
end
|
352
|
+
nfound += 1
|
353
|
+
end
|
354
|
+
end
|
355
|
+
end
|
356
|
+
|
357
|
+
sigs.delete_if{ |sig| sig.re.empty? }
|
358
|
+
end
|
359
|
+
|
360
|
+
def _name2wordonly name
|
361
|
+
name.downcase.split(/[^a-z0-9_.]+/).join(' ').strip
|
362
|
+
end
|
363
|
+
|
364
|
+
def optimize_names sigs
|
365
|
+
# replaces all duplicate names with references to one name
|
366
|
+
# saves ~30k out of ~200k mem
|
367
|
+
h = {}
|
368
|
+
|
369
|
+
# find shortest names
|
370
|
+
sigs.each do |sig|
|
371
|
+
t = _name2wordonly(sig.name)
|
372
|
+
if h[t]
|
373
|
+
# keep shortest name
|
374
|
+
if h[t] != sig.name
|
375
|
+
#print "[d] #{[h[t], sig.name].inspect} -> "
|
376
|
+
h[t] = [h[t], sig.name].sort_by(&:size).first
|
377
|
+
#puts h[t]
|
378
|
+
else
|
379
|
+
# fully identical names
|
380
|
+
end
|
381
|
+
else
|
382
|
+
h[t] = sig.name
|
383
|
+
end
|
384
|
+
end
|
385
|
+
|
386
|
+
# assign names back to sigs
|
387
|
+
sigs.each{ |sig| sig.name = h[_name2wordonly(sig.name)] }
|
388
|
+
|
389
|
+
puts "[.] sigs merge: #{h.size} unique names"
|
390
|
+
end
|
391
|
+
|
392
|
+
def optimize sigs
|
393
|
+
optimize_names sigs
|
394
|
+
|
395
|
+
# XXX no optimize from now, prefer more precise sigs
|
396
|
+
#print "[.] sigs merge: #{sigs.size}"; _optimize(sigs); puts " -> #{sigs.size}"
|
397
|
+
|
398
|
+
# try to merge signatures with same name, size & ep_only
|
399
|
+
sigs.group_by{ |sig| [sig.re.size, sig.name, sig.ep_only] }.
|
400
|
+
values.each do |a|
|
401
|
+
next if a.size == 1
|
402
|
+
if merged_re = _merge(a)
|
403
|
+
a.first.re = merged_re
|
404
|
+
a[1..-1].each{ |sig| sig.re = nil }
|
405
|
+
end
|
406
|
+
end
|
407
|
+
print "[.] sigs merge: #{sigs.size}"; sigs.delete_if{ |x| x.re.nil? }; puts " -> #{sigs.size}"
|
408
|
+
|
409
|
+
|
410
|
+
# 361 entries of ["VMProtect v1.25 (PolyTech)", true, "h....\xE8...."])
|
411
|
+
sigs.group_by{ |sig|
|
412
|
+
[sig.name, sig.ep_only, sig.re[0,10].join]
|
413
|
+
}.each do |k,entries|
|
414
|
+
next if entries.size < 10
|
415
|
+
#printf "%5d %s\n", entries.size, k
|
416
|
+
prefix = entries.first.re[0,10]
|
417
|
+
infix = entries.map{ |sig| sig.re[10..-1] }
|
418
|
+
|
419
|
+
entries.first.re = prefix + [OrBlock.new(infix)]
|
420
|
+
entries.first.size = entries.map(&:size).max
|
421
|
+
|
422
|
+
entries[1..-1].each{ |sig| sig.re = nil }
|
423
|
+
end
|
424
|
+
print "[.] sigs merge: #{sigs.size}"; sigs.delete_if{ |x| x.re.nil? }; puts " -> #{sigs.size}"
|
425
|
+
|
426
|
+
|
427
|
+
# # merge signatures with same prefix & suffix
|
428
|
+
# # most ineffecient part :)
|
429
|
+
# sigs.group_by{ |sig|
|
430
|
+
# [sig.name, sig.ep_only, sig.re.index{ |x| x.is_a?(Array)}]
|
431
|
+
# }.values.each do |a|
|
432
|
+
# next if a.size == 1
|
433
|
+
# next unless idx = a.first.re.index{ |x| x.is_a?(Array) }
|
434
|
+
# a.group_by{ |sig| [sig.re[0...idx], sig.re[(idx+1)..-1]] }.each do |k,entries|
|
435
|
+
# # prefix | infix | suffix
|
436
|
+
# # s o m [[b r e r o] [e w h a t]] h e r e
|
437
|
+
# prefix, suffix = k
|
438
|
+
# infix = entries.map{ |sig| sig.re[idx] }
|
439
|
+
# #infix = [['f','o','o']]
|
440
|
+
# merged_re = prefix + infix + suffix
|
441
|
+
# max_size = entries.map(&:size).max
|
442
|
+
# entries.each{ |sig| sig.re = merged_re; sig.size = max_size }
|
443
|
+
# end
|
444
|
+
# end
|
445
|
+
# print "[.] sigs merge: #{sigs.size}"; sigs.uniq!; puts " -> #{sigs.size}"
|
446
|
+
|
447
|
+
# stats
|
448
|
+
# aa = []
|
449
|
+
# 6.upto(20) do |len|
|
450
|
+
# sigs.group_by{ |sig| [sig.re[0,len].join, sig.name, sig.ep_only] }.each do |a,b|
|
451
|
+
# aa << [b.size, a[0], [b.map(&:size).min, b.map(&:size).max].join(' .. ') ] if b.size > 2
|
452
|
+
# end
|
453
|
+
# end
|
454
|
+
# aa.sort_by(&:first).each do |sz,prefix,name|
|
455
|
+
# printf "%5d %-50s %s\n", sz, prefix.inspect, name
|
456
|
+
# end
|
457
|
+
|
458
|
+
sigs
|
459
|
+
end
|
460
|
+
|
461
|
+
# range of common difference between N given sigs
|
462
|
+
def _diff res
|
463
|
+
raise "diff sizes" if res.map(&:size).uniq.size != 1
|
464
|
+
size = res.first.size
|
465
|
+
|
466
|
+
dstart = nil
|
467
|
+
dend = size - 1
|
468
|
+
prev_eq = true
|
469
|
+
|
470
|
+
size.times do |i|
|
471
|
+
eq = res.map{ |re| re[i] }.uniq.size == 1
|
472
|
+
if eq != prev_eq
|
473
|
+
if eq
|
474
|
+
# end of current diff
|
475
|
+
dend = i-1
|
476
|
+
else
|
477
|
+
# start of new diff
|
478
|
+
return nil if dstart # return nil if it's a 2nd diff
|
479
|
+
dstart = i
|
480
|
+
end
|
481
|
+
end
|
482
|
+
prev_eq = eq
|
483
|
+
end
|
484
|
+
dstart ||= 0
|
485
|
+
r = dstart..dend
|
486
|
+
r == (0..(size-1)) ? nil : r
|
487
|
+
end
|
488
|
+
|
489
|
+
# merge array of signatures into one signature
|
490
|
+
def _merge sigs
|
491
|
+
sizes = sigs.map(&:re).map(&:size)
|
492
|
+
|
493
|
+
if sizes.uniq.size != 1
|
494
|
+
puts "[?] wrong sizes: #{sizes.inspect}"
|
495
|
+
return nil
|
496
|
+
end
|
497
|
+
|
498
|
+
res = sigs.map(&:re)
|
499
|
+
diff = _diff res
|
500
|
+
return nil unless diff
|
501
|
+
|
502
|
+
ref = res.first
|
503
|
+
ref[0...diff.first] + [OrBlock.new(res.map{ |re| re[diff] })] + ref[(diff.last+1)..-1]
|
504
|
+
end
|
505
|
+
end
|
506
|
+
end
|
507
|
+
end
|