pedump 0.4.14 → 0.4.15

data/lib/pedump/sig_parser.rb

@@ -6,9 +6,14 @@ class PEdump
     TEXT_SIGS_FILES = [
       File.join(DATA_ROOT, "data", "userdb.txt"),
       File.join(DATA_ROOT, "data", "signatures.txt"),
-      File.join(DATA_ROOT, "data", "fs.txt")
+      File.join(DATA_ROOT, "data", "jc-userdb.txt"),
+      File.join(DATA_ROOT, "data", "fs.txt"),         # has special parse options!
     ]
 
+    SPECIAL_PARSE_OPTIONS = {
+      File.join(DATA_ROOT, "data", "fs.txt") => {:fix1 => true}
+    }
+
     class OrBlock < Array; end
 
     class << self
@@ -19,19 +24,20 @@ class PEdump
         sigs = {}; sig = nil
 
         args[:fnames].each do |fname|
-          n0 = sigs.size
+          n0 = sigs.size; add_sig_args = args.dup
+          add_sig_args.merge!(SPECIAL_PARSE_OPTIONS[fname] || {})
           File.open(fname,'r:utf-8') do |f|
             while line = f.gets
               case line.strip
               when /^[<;#]/, /^$/ # comments & blank lines
                 next
               when /^\[(.+)=(.+)\]$/
-                _add_sig(sigs, Packer.new($1, $2, true), args )
-              when /^\[([^=]+)\]$/
+                _add_sig(sigs, Packer.new($1, $2, true), add_sig_args )
+              when /^\[([^=]+)\](\s+\/\/.+)?$/
                 sig = Packer.new($1)
               when /^signature = (.+)$/
                 sig.re = $1
-                _add_sig(sigs, sig, args)
+                _add_sig(sigs, sig, add_sig_args)
               when /^ep_only = (.+)$/
                 sig.ep_only = ($1.strip.downcase == 'true')
               else raise line
@@ -61,7 +67,13 @@ class PEdump
               when /\A[a-f0-9]{2}\Z/i
                 x = x.to_i(16).chr
                 bins[sig] << x
-                args[:raw] ? x : Regexp::escape(x)
+                if args[:raw]
+                  x
+                elsif args[:raword]
+                  x.ord
+                else
+                  Regexp::escape(x)
+                end
               else
                 puts "[?] unknown re element: #{x.inspect} in #{sig.inspect}" if args[:verbose]
                 "BAD_RE"
@@ -72,10 +84,10 @@ class PEdump
             a = sig.name.split(/-+>/,2).map(&:strip)
             sig.name = "#{a[0]} (#{a[1]})"
           end
-          sig.re.pop while sig.re.last == '??'
+          sig.re.pop while sig.re && sig.re.last == '??'
         end
         sigs.delete_if{ |sig| !sig.re || sig.re.index('BAD_RE') }
-        return sigs if args[:raw]
+        return sigs if args[:raw] || args[:raword]
 
 #        require 'awesome_print'
 #        bins.each do |bin_sig, bin|
@@ -125,6 +137,9 @@ class PEdump
         # bad sigs
         return if sig.re[/\A538BD833C0A30:::::/]
         return if sig.name == "Name of the Packer v1.0"
+        return if sig.name == "Alias PIX/Vivid IMG Graphics format"
+        return if sig.name == "JAR Archive"
+        return if sig.name == "Turbo / Borland Pascal v7.x Unit"
         return if sig.re == "54 68 69 73 20 70 72 6F 67 72 61 6D 20 63 61 6E 6E 6F 74 20 62 65 20 72 75 6E 20 69 6E 20 44 4F 53 20 6D 6F" # dos stub
 
         sig.name.sub!(/^\*\s+/,    '')
@@ -135,10 +150,47 @@ class PEdump
         sig.name.sub! 'RLP ','RLPack '
         sig.name.sub! '.beta', ' beta'
         sig.name.sub! '(com)','[com]'
+        sig.name.gsub!(/ V(\d)/, " v\\1") # V1.1 -> v1.1
         sig.name = sig.name.split(/\s*-+>\s*/).join(' -> ') # fix spaces around '->'
+        sig.name = sig.name.split(' ').delete_if do |x|
+          # delete words: vX.X, v?.?, ?.?, x.x
+          x =~ /\Av?[?x]\.[?x]\Z/i
+        end.join(' ')
 
         sig.re = sig.re.strip.upcase.tr(':','?')
         sig.re = sig.re.scan(/../).join(' ') if sig.re.split.first.size > 2
+
+        # sig contains entirely zeroes or masks or only both
+        a_bad = [%w'00', %w'??', %w'00 ??', %w'90', %w'90 ??']
+        # ?a, 0? => ??, ??
+        a_cur = sig.re.split.map{ |x| x['?'] ? '??' : x }.uniq.sort
+        return if a_bad.include?(a_cur)
+
+        # first byte is unique and all others are zeroes or masks
+        a_cur = sig.re.split[1..-1].map{ |x| x['?'] ? '??' : x }.uniq.sort
+        return if a_bad.include?(a_cur)
+
+        # too short signatures
+#        if sig.re.split.delete_if{ |x| x['?'] }.size < 6
+#          require 'awesome_print'
+#          puts sig.inspect.red
+#        end
+
+        # fs.txt contains a lot of signatures that copied from other sources
+        # BUT have all 01 replaced with '??'
+        # // replaced the file with filtered one (see 'fs-good' below) // zzz
+        if args[:fix1]
+          sigs.keys.each do |re|
+            if re.gsub("01","??") == sig.re
+              puts "[.] fix1: rejecting #{sig.name} - already present with 01 in place" if args[:verbose]
+              return
+            end
+          end
+#          File.open("fs-good.txt","a") do |f|
+#            f << "[#{sig.name}=#{sig.re.tr(' ','')}]\n"
+#          end
+        end
+
         if sigs[sig.re]
           a = [sig, sigs[sig.re]].map{ |x| x.name.upcase.split('->').first.tr('V ','') }
           return if a[0][a[1]] || a[1][a[0]]
@@ -161,7 +213,6 @@ class PEdump
         d.map! do |x|
           x - [
             'EXE','[EXE]',
-            'vx.x','v?.?',
             'DLL','(DLL)','[DLL]',
             '[LZMA]','(LZMA)','LZMA',
             '-','~','(pack)','(1)','(2)',
@@ -264,14 +315,41 @@ class PEdump
         sigs.delete_if{ |sig| sig.re.empty? }
       end
 
-      def optimize sigs
+      def _name2wordonly name
+        name.downcase.split(/[^a-z0-9_.]+/).join(' ').strip
+      end
+
+      def optimize_names sigs
         # replaces all duplicate names with references to one name
         # saves ~30k out of ~200k mem
         h = {}
+
+        # find shortest names
         sigs.each do |sig|
-          sig.name = (h[sig.name] ||= sig.name)
+          t = _name2wordonly(sig.name)
+          if h[t]
+            # keep shortest name
+            if h[t] != sig.name
+              #print "[d] #{[h[t], sig.name].inspect} -> "
+              h[t] = [h[t], sig.name].sort_by(&:size).first
+              #puts h[t]
+            else
+              # fully identical names
+            end
+          else
+            h[t] = sig.name
+          end
         end
 
+        # assign names back to sigs
+        sigs.each{ |sig| sig.name = h[_name2wordonly(sig.name)] }
+
+        puts "[.] sigs merge: #{h.size} unique names"
+      end
+
+      def optimize sigs
+        optimize_names sigs
+
         print "[.] sigs merge: #{sigs.size}"; _optimize(sigs); puts  " -> #{sigs.size}"
 
         # try to merge signatures with same name, size & ep_only

data/lib/pedump/version.rb

@@ -2,7 +2,7 @@ class PEdump
   module Version
     MAJOR = 0
     MINOR = 4
-    PATCH = 14
+    PATCH = 15
     BUILD = nil
 
     STRING = [MAJOR, MINOR, PATCH, BUILD].compact.join('.')

data/pedump.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = "pedump"
-  s.version = "0.4.14"
+  s.version = "0.4.15"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Andrey \"Zed\" Zaikin"]
-  s.date = "2012-12-17"
+  s.date = "2012-12-28"
   s.description = "dump headers, sections, extract resources of win32 PE exe,dll,etc"
   s.email = "zed.0xff@gmail.com"
   s.executables = ["pedump"]
@@ -29,6 +29,7 @@ Gem::Specification.new do |s|
     "VERSION",
     "bin/pedump",
     "data/fs.txt",
+    "data/jc-userdb.txt",
     "data/sig.bin",
     "data/signatures.txt",
     "data/userdb.txt",
@@ -104,7 +105,6 @@ Gem::Specification.new do |s|
       s.add_development_dependency(%q<bundler>, [">= 0"])
       s.add_development_dependency(%q<jeweler>, [">= 0"])
       s.add_development_dependency(%q<what_methods>, [">= 0"])
-      s.add_development_dependency(%q<looksee>, [">= 0"])
     else
       s.add_dependency(%q<multipart-post>, ["~> 1.1.4"])
       s.add_dependency(%q<progressbar>, [">= 0"])
@@ -113,7 +113,6 @@ Gem::Specification.new do |s|
       s.add_dependency(%q<bundler>, [">= 0"])
       s.add_dependency(%q<jeweler>, [">= 0"])
       s.add_dependency(%q<what_methods>, [">= 0"])
-      s.add_dependency(%q<looksee>, [">= 0"])
     end
   else
     s.add_dependency(%q<multipart-post>, ["~> 1.1.4"])
@@ -123,7 +122,6 @@ Gem::Specification.new do |s|
     s.add_dependency(%q<bundler>, [">= 0"])
     s.add_dependency(%q<jeweler>, [">= 0"])
     s.add_dependency(%q<what_methods>, [">= 0"])
-    s.add_dependency(%q<looksee>, [">= 0"])
   end
 end
 

data/spec/sig_spec.rb

@@ -31,6 +31,11 @@ describe "PEdump::Packer" do
         next unless row =~ /^\[(.*)=(.*)\]$/
         s = ''
         title,hexstring = $1,$2
+
+        # bad sigs
+        next if hexstring == '909090909090909090909090909090909090909090909090909090909090909090909090'
+        next if hexstring == 'E9::::0000000000000000'
+
         (hexstring.size/2).times do |i|
           c = hexstring[i*2,2]
           if c == '::'
@@ -52,7 +57,7 @@ describe "PEdump::Packer" do
 #            puts "\t= #{x}"
 #          end
         else
-          puts "[?] #{title}"
+          puts "[?] #{title}: #{hexstring}"
           n += 1
         end
       end

metadata

@@ -1,144 +1,128 @@
 --- !ruby/object:Gem::Specification
 name: pedump
 version: !ruby/object:Gem::Version
-  version: 0.4.14
   prerelease: 
+  version: 0.4.15
 platform: ruby
 authors:
 - Andrey "Zed" Zaikin
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-12-17 00:00:00.000000000 Z
+date: 2012-12-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :runtime
   name: multipart-post
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.1.4
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
         version: 1.1.4
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :runtime
   name: progressbar
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :runtime
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :development
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :development
   name: bundler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :development
   name: jeweler
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
     none: false
+  version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 - !ruby/object:Gem::Dependency
+  prerelease: false
+  type: :development
   name: what_methods
   requirement: !ruby/object:Gem::Requirement
-    none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: looksee
-  requirement: !ruby/object:Gem::Requirement
     none: false
-    requirements:
-    - - ! '>='
-      - !ruby/object:Gem::Version
-        version: '0'
-  type: :development
-  prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
+    none: false
 description: dump headers, sections, extract resources of win32 PE exe,dll,etc
 email: zed.0xff@gmail.com
 executables:
@@ -159,6 +143,7 @@ files:
 - VERSION
 - bin/pedump
 - data/fs.txt
+- data/jc-userdb.txt
 - data/sig.bin
 - data/signatures.txt
 - data/userdb.txt
@@ -224,20 +209,20 @@ rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
       segments:
       - 0
-      hash: -4041753216980716747
-required_rubygems_version: !ruby/object:Gem::Requirement
+      hash: -4443824394784718020
   none: false
+required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
+  none: false
 requirements: []
 rubyforge_project: 
 rubygems_version: 1.8.24