memorandom 0.0.1

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+gem 'credit_card_validator'
+

data/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2014 Rapid7
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,41 @@
+Memorandom
+==
+
+Memorandom provides a command-line utility and class library from extracting secrets from binary files. Common use cases include extracting encryption keys from memory dumps and identifying sensitive data stored in block devices.
+
+
+Installation
+--
+    $ git clone https://github.com/rapid7/memorandom
+    $ cd memorandom
+    $ bundle install
+    
+Usage
+--
+    $ bin/memorandom.rb --help
+    
+    Usage: bin/memorandom.rb [options] file1 file2 ... fileN
+    Extracts interesting data from binary files
+
+    Options
+        -p [plugin1,plugin2,plugin3...], Specify a list of plugin names to use, otherwise use all plugins
+            --plugins
+        -o, --output [directory]         Specify the directory in which to store found data
+        -w, --window [number]            Specify the number of kilobytes to scan at once (default: 1024).
+        -x, --overlap [number]           Specify the number of kilobytes to overlap between windows (default: 4).
+        -l, --list-plugins               List all of the available plugins
+        -h, --help                       Show this message.
+
+
+Memorandom can search files, block devices, and standard input for interesting things and automatically extract and save these into the specified output directory (--output). This makes it useful for processing memory dumps, network traffic logs, hard disk images, or entire filesystems. 
+
+
+Memorandom uses plugins to scan for specific types of data within the target files. Ny default, all plugins are enabled, which can lead to noisy and slow output. For small files, all plugins are usually fine, but when processing large amounts of data, it is better to limit the search to specific plugins.
+
+The example below will only scan for PEM-encoded data in the target files
+
+    $ bin/memorandom.rb -p pem ~/.ssh/*
+    [+] file:/home/dev/.ssh/ec2.pem PEM@0 ("-----BEGIN RSA PRIVATE KEY-----\r"...)
+    [+] file:/home/dev/.ssh/id_dsa PEM@0 ("-----BEGIN DSA PRIVATE KEY-----\n"...)
+    [+] file:/home/dev/.ssh/id_rsa PEM@0 ("-----BEGIN RSA PRIVATE KEY-----\n"...)
+

data/bin/memorandom.rb

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+$:.unshift(File.expand_path(File.join(File.dirname(__FILE__), "..", "lib")))
+require 'optparse'
+require 'ostruct'
+require 'memorandom'
+
+options = {}
+
+option_parser = OptionParser.new do |opts|
+  opts.banner = "Usage: #{$0} [options] file1 file2 ... fileN"
+  opts.separator "Extracts interesting data from binary files"
+  opts.separator ""
+  opts.separator "Options"
+
+  opts.on("-p", "--plugins [plugin1,plugin2,plugin3...]", 
+          "Specify a list of plugin names to use, otherwise use all plugins" 
+    ) do |plugins|
+    options[:plugins] = plugins.split(",").map{ |x| x.downcase.strip }
+  end
+
+  opts.on("-o", "--output [directory]",
+          "Specify the directory in which to store found data") do |out|
+    options[:output] = out
+  end
+
+  opts.on("-w", "--window [number]",
+          "Specify the number of kilobytes to scan at once (default: 1024).") do |num|
+    options[:window] = (num.to_i == 0) ? (1024*1024) : (num.to_i * 1024)
+  end
+
+  opts.on("-x", "--overlap [number]",
+          "Specify the number of kilobytes to overlap between windows (default: 4).") do |num|
+    options[:overlap] = (num.to_i == 0) ? (4*1024) : (num.to_i * 1024)
+  end
+
+  opts.on("-l", "--list-plugins",
+          "List all of the available plugins") do
+    puts ""
+    puts "Memorandom Plugins"
+    puts "==============="
+
+    Memorandom::PluginManager.plugins.each_pair do |name, klass|
+      puts "   * #{name.ljust(24)} #{klass.description}"
+    end
+    puts ""
+    exit
+  end
+
+  opts.on("-h", "--help", "Show this message.") do
+    puts opts
+    exit
+  end
+end
+option_parser.parse!(ARGV)
+
+if ARGV.count < 1
+  puts option_parser
+  exit
+end
+
+scanner = Memorandom::Scanner.new(options)
+if scanner.plugins.length == 0
+  $stderr.puts "Error: No valid plugins have been selected"
+  exit(1)
+end
+
+ARGV.each do |target|
+  scanner.scan(target)
+end

data/lib/memorandom.rb

@@ -0,0 +1,5 @@
+require 'memorandom/version'
+require 'memorandom/plugins'
+require 'memorandom/scanner'
+
+Memorandom::PluginManager.load_plugins

data/lib/memorandom/plugins.rb

@@ -0,0 +1,31 @@
+module Memorandom
+  class PluginManager
+    @@plugins = {}
+
+    def self.register(name, klass)
+      @@plugins[name.downcase.gsub(/\s+/, '_')] = klass
+    end
+
+    def self.plugins
+      @@plugins
+    end
+
+    def self.load_plugins
+      Memorandom::Plugins.constants.each do |c|
+        register(c.to_s, Memorandom::Plugins.const_get(c))
+      end
+    end
+  
+  end
+end
+
+require 'memorandom/plugins/template'
+
+require 'memorandom/plugins/pem'
+require 'memorandom/plugins/der'
+require 'memorandom/plugins/rsa'
+require 'memorandom/plugins/aes'
+require 'memorandom/plugins/capi'
+require 'memorandom/plugins/url_params'
+require 'memorandom/plugins/hashes'
+require 'memorandom/plugins/cc'

data/lib/memorandom/plugins/aes.rb

@@ -0,0 +1,188 @@
+module Memorandom
+module Plugins
+class AES < PluginTemplate
+
+  @@description = "This plugin looks for AES encryption keys (128/256)"
+  @@confidence  = 0.50
+
+  #
+  # This code is a ruby implementation of AESKeyFind (C) 2008-07-18 Nadia Heninger and Ariel Feldman
+  # It is approximately 360 times slower than the C implementation :(
+  #
+
+  AES_KEYBLOCK_MIN = 176
+  AES_XOR_LIMIT    = 10
+  AES_SBOX =  [
+    # 0     1    2      3     4    5     6     7      8    9     A      B    C     D     E     F
+    0x63, 0x7c, 0x77, 0x7b, 0xf2, 0x6b, 0x6f, 0xc5, 0x30, 0x01, 0x67, 0x2b, 0xfe, 0xd7, 0xab, 0x76, #0
+    0xca, 0x82, 0xc9, 0x7d, 0xfa, 0x59, 0x47, 0xf0, 0xad, 0xd4, 0xa2, 0xaf, 0x9c, 0xa4, 0x72, 0xc0, #1
+    0xb7, 0xfd, 0x93, 0x26, 0x36, 0x3f, 0xf7, 0xcc, 0x34, 0xa5, 0xe5, 0xf1, 0x71, 0xd8, 0x31, 0x15, #2
+    0x04, 0xc7, 0x23, 0xc3, 0x18, 0x96, 0x05, 0x9a, 0x07, 0x12, 0x80, 0xe2, 0xeb, 0x27, 0xb2, 0x75, #3
+    0x09, 0x83, 0x2c, 0x1a, 0x1b, 0x6e, 0x5a, 0xa0, 0x52, 0x3b, 0xd6, 0xb3, 0x29, 0xe3, 0x2f, 0x84, #4
+    0x53, 0xd1, 0x00, 0xed, 0x20, 0xfc, 0xb1, 0x5b, 0x6a, 0xcb, 0xbe, 0x39, 0x4a, 0x4c, 0x58, 0xcf, #5
+    0xd0, 0xef, 0xaa, 0xfb, 0x43, 0x4d, 0x33, 0x85, 0x45, 0xf9, 0x02, 0x7f, 0x50, 0x3c, 0x9f, 0xa8, #6
+    0x51, 0xa3, 0x40, 0x8f, 0x92, 0x9d, 0x38, 0xf5, 0xbc, 0xb6, 0xda, 0x21, 0x10, 0xff, 0xf3, 0xd2, #7
+    0xcd, 0x0c, 0x13, 0xec, 0x5f, 0x97, 0x44, 0x17, 0xc4, 0xa7, 0x7e, 0x3d, 0x64, 0x5d, 0x19, 0x73, #8
+    0x60, 0x81, 0x4f, 0xdc, 0x22, 0x2a, 0x90, 0x88, 0x46, 0xee, 0xb8, 0x14, 0xde, 0x5e, 0x0b, 0xdb, #9
+    0xe0, 0x32, 0x3a, 0x0a, 0x49, 0x06, 0x24, 0x5c, 0xc2, 0xd3, 0xac, 0x62, 0x91, 0x95, 0xe4, 0x79, #A
+    0xe7, 0xc8, 0x37, 0x6d, 0x8d, 0xd5, 0x4e, 0xa9, 0x6c, 0x56, 0xf4, 0xea, 0x65, 0x7a, 0xae, 0x08, #B
+    0xba, 0x78, 0x25, 0x2e, 0x1c, 0xa6, 0xb4, 0xc6, 0xe8, 0xdd, 0x74, 0x1f, 0x4b, 0xbd, 0x8b, 0x8a, #C
+    0x70, 0x3e, 0xb5, 0x66, 0x48, 0x03, 0xf6, 0x0e, 0x61, 0x35, 0x57, 0xb9, 0x86, 0xc1, 0x1d, 0x9e, #D
+    0xe1, 0xf8, 0x98, 0x11, 0x69, 0xd9, 0x8e, 0x94, 0x9b, 0x1e, 0x87, 0xe9, 0xce, 0x55, 0x28, 0xdf, #E
+    0x8c, 0xa1, 0x89, 0x0d, 0xbf, 0xe6, 0x42, 0x68, 0x41, 0x99, 0x2d, 0x0f, 0xb0, 0x54, 0xbb, 0x16  #F
+  ]
+
+  AES_RCON = [
+    0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 
+    0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 
+    0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 
+    0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8,
+    0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef,
+    0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 
+    0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1b,
+    0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3,
+    0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 
+    0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04, 0x08, 0x10, 0x20,
+    0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 0xc6, 0x97, 0x35,
+    0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd, 0x61, 0xc2, 0x9f,
+    0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb, 0x8d, 0x01, 0x02, 0x04,
+    0x08, 0x10, 0x20, 0x40, 0x80, 0x1b, 0x36, 0x6c, 0xd8, 0xab, 0x4d, 0x9a, 0x2f, 0x5e, 0xbc, 0x63, 
+    0xc6, 0x97, 0x35, 0x6a, 0xd4, 0xb3, 0x7d, 0xfa, 0xef, 0xc5, 0x91, 0x39, 0x72, 0xe4, 0xd3, 0xbd,
+    0x61, 0xc2, 0x9f, 0x25, 0x4a, 0x94, 0x33, 0x66, 0xcc, 0x83, 0x1d, 0x3a, 0x74, 0xe8, 0xcb
+  ]
+
+  BYTE_VALUES = Array.new([0]*256)
+  ENTROPIC_MIN = 8
+
+  M1 = 0x55555555
+  M2 = 0x33333333
+  M4 = 0x0f0f0f0f
+
+  # Perform the AES key core operation on a word.
+  # (Assumes the standard byte order.)
+  def aes_key_core(k, i)
+    t = 0
+    0.upto(3) do |j|
+      t = aes_set_byte(t, (j-1) % 4, AES_SBOX[aes_get_byte(k, j)])
+    end
+    aes_set_byte(t, 0, aes_get_byte(t,0) ^ AES_RCON[i])
+  end
+
+  # Run each byte of a word through the sbox separately for word 4 of 256-bit AES.
+  def aes_sbox_bytes(k)
+    r = 0
+    0.upto(3) do |j|
+      r = aes_set_byte(r, j, AES_SBOX[aes_get_byte(k,j)])
+    end
+    r
+  end
+
+  # Return bit n of vector.
+  def aes_bit(vector, n)
+    (vector >> n) & 1
+  end
+
+  # Set byte n of vector to val.
+  def aes_set_byte(vector, n, val)
+    (vector & ~(0xff << (8*n))) | (val << (8*n))
+  end
+
+  # Return byte n of vector.
+  def aes_get_byte(vector, n)
+    (vector >> (8*n)) & 0xff
+  end
+
+  # Return the number of bits in x that are 1.
+  def aes_popcount(x)
+    x -= (x >> 1) & M1
+    x = (x & M2) + ((x >> 2) & M2)
+    x = (x + (x >> 4)) & M4
+    x += x >> 8
+    (x + (x >> 16)) & 0x3f
+  end
+
+  # Identifies byte windows that are "entropic" (< limit repeats of any byte)
+  # Algorithm adapted from aeskeyfind.c
+  def entropic(buff, window, limit)
+    bytes = buff.unpack("C*")
+    index = 0
+
+    while index < (bytes.length - window)
+      counts = BYTE_VALUES.dup
+      index.upto(index+window) {|offset| counts[ bytes[offset] ] += 1 }
+      mvalue = counts.max
+
+      if mvalue < limit
+        yield(index)
+        # Key-hunting time, skip ahead by one byte only
+        index += 1
+      else
+        # Not entropic, skip ahead by at least max - limit
+        index += ((mvalue + 1) - limit)
+      end
+    end
+  end
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    blen = buffer.length
+    entropic(buffer, AES_KEYBLOCK_MIN, ENTROPIC_MIN) do |i|
+
+      # Exit if we dont have at least 240 bytes left
+      return if (i + 240) > blen
+
+      # Create a byte map to work in both little and big endian formats
+      #["V", "N"].each do |endian|
+
+      ["V"].each do |endian|
+        bmap = buffer[i, 240].unpack("#{endian}*")
+
+        # Check distance from 256-bit AES key
+        xor_count_256 = 0
+        1.upto(7) do |row|
+          0.upto(7) do |column|
+            break if (row == 7 and column == 4)
+            case column
+            when 0
+              xor_count_256 += aes_popcount( aes_key_core( bmap[8*row-1], row) ^ bmap[8*(row-1)] ^ bmap[8*row] )
+            when 4
+              xor_count_256 += aes_popcount( aes_sbox_bytes( bmap[8*row+3]) ^ bmap[8*(row-1)+4] ^ bmap[8*row+4])
+            else
+              xor_count_256 += aes_popcount( bmap[8*row+column-1] ^ bmap[8*(row-1)+column] ^ bmap[8*row + column])
+            end
+          end
+        end
+
+        # We found a possible AES-256 key
+        unless xor_count_256 > AES_XOR_LIMIT
+          report_hit(:type => "Key(AES-256)", :data => bmap[0, 256/32].pack("#{endian}*").unpack("C*").map{|x| "%.2x" % x }.join, :offset => source_offset + i)
+          next
+        end
+
+        # Check distance from 128-bit AES key
+        xor_count_128 = 0
+        1.upto(10) do |row|
+          0.upto(3) do |column|
+            before_count = xor_count_128
+            case column
+            when 0 
+              xor_count_128 += aes_popcount( aes_key_core( bmap[4*row-1],row) ^ bmap[4*(row-1)] ^ bmap[4*row])
+            else
+              xor_count_128 += aes_popcount( (bmap[4*row + column-1] ^ bmap[4*(row-1)+column]) ^ bmap[4*row + column])
+            end
+          end
+        end
+
+        #  We found a possible AES-128 key
+        unless xor_count_128 > AES_XOR_LIMIT
+          report_hit(:type => "Key(AES-128)", :data => bmap[0, 128/32].pack("#{endian}*").unpack("C*").map{|x| "%.2x" % x }.join, :offset => source_offset + i)
+        end
+      end
+    end
+
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/capi.rb

@@ -0,0 +1,62 @@
+module Memorandom
+module Plugins
+class CAPI < PluginTemplate
+
+  require 'openssl'
+
+  @@description = "This plugin looks for Microsoft CryptoAPI encryption keys in memory (PRIVATEBLOB)"
+  @@confidence  = 0.90
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    buffer.scan(
+      /[\x06\x07]\x02.{6}(?:RSA1|RSA2|DSS1|DSS2).{20}/
+    ).each do |m|
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      next unless last_offset 
+
+      # Attempt to parse the key at the specified offset
+      key_candidate = buffer[last_offset, 20000]
+      key_type = ""
+      key      = nil
+
+      bits = key_candidate.unpack("CCA6A4V")
+      key_type << ( bits[3] + "-" + bits[4].to_s + "-" )
+      key_type << ( (bits[0] == 0x07) ? "Private" : "Public" )
+
+      key_length = 0
+      nbyte  = (bits[4] +  7) >> 3
+      hnbyte = (bits[4] + 15) >> 4 
+
+      # DSA
+      if bits[3].index('DSS')
+        if bits[0] == 0x06
+          # Expected length: 20 for q + 3 components bitlen each + 24 for seed structure.
+          key_length = 44 + (3 * nbyte)
+        else
+          # Expected length: 20 for q, priv, 2 bitlen components + 24 for seed structure.
+          key_length = 64 + (2 * nbyte)
+        end
+      # RSA
+      else
+        if bits[0] == 0x06
+          # Expected length: 4 for 'e' + 'n'
+          key_length = 4 + nbyte
+        else
+          # Expected length: 4 for 'e' and 7 other components. 2 components are bitlen size, 5 are bitlen/2
+          key_length = 4 + (2 * nbyte) + (5 * hnbyte)
+        end
+      end
+
+      key = buffer[last_offset, key_length + 16]
+      next unless key.length == (key_length+16)
+
+      report_hit(:type => "CAPI-#{key_type}", :data => key, :offset => source_offset + last_offset)
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/cc.rb

@@ -0,0 +1,36 @@
+module Memorandom
+module Plugins
+class CC < PluginTemplate
+
+  # Use the credit_card_validator gem for luhn checks and card verification
+  require 'credit_card_validator'
+
+  @@description = "This plugin looks for credit card numbers"
+  @@confidence  = 0.10
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    buffer.scan(
+      # Look for credit card numbers in various formats
+      /(?:^|\D)([\d \-]{12,32})(?:$|\D)/
+    ).each do |m|
+      matched = m.first
+
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(matched)
+      next unless last_offset
+
+      # Clean out any non-digits and validate the card
+      cleaned = matched.gsub(/[^\d]+/, '')
+      next unless CreditCardValidator::Validator.valid?(cleaned)
+      cc_type = CreditCardValidator::Validator.card_type(cleaned)
+      cc_type = cc_type.split('_').map{|x| x.capitalize }.join
+
+      report_hit(:type => "CreditCard(#{cc_type})", :data => cleaned, :offset => source_offset + last_offset)
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/der.rb

@@ -0,0 +1,43 @@
+module Memorandom
+module Plugins
+class DER < PluginTemplate
+
+  require 'openssl'
+
+  @@description = "This plugin looks for DER-encoded encryption keys (RSA/DSA/EC)"
+  @@confidence  = 0.90
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    buffer.scan(
+      # Look for a DER record start (0x30), a length value, and a version marker. 
+      # This identifies RSA, DSA, and EC keys
+      /\x30.{1,5}\x02\x01(?:\x00\x02|\x01\x04)/m
+    ).each do |m|
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      next unless last_offset 
+
+      # Attempt to parse the key at the specified offset
+      key_candidate = buffer[last_offset, 20000]
+      key_type = nil
+      key      = nil
+
+      [:RSA, :DSA, :EC ].each do |ktype|
+        next unless OpenSSL::PKey.const_defined?(ktype)
+        key_type = ktype
+        key = OpenSSL::PKey.const_get(ktype).new(key_candidate) rescue nil
+        break if key
+      end
+
+      # Ignore this if OpenSSL could not parse out a valid key
+      next unless key
+
+      report_hit(:type => "#{key_type}", :data => key.to_pem, :offset => source_offset + last_offset)
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/hashes.rb

@@ -0,0 +1,36 @@
+module Memorandom
+module Plugins
+class Hashes < PluginTemplate
+
+  @@description = "This plugin looks for common hash formats"
+  @@confidence  = 0.10
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    # Unix password hash formats
+    buffer.scan(
+      /[a-z0-9_]+:\$\d+\$[$a-z0-9\.\/]+:\d+:\d+:\d+[a-z0-9 :]*/mi
+    ).each do |m|
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      report_hit(:type => 'UnixHash', :data => m, :offset => source_offset + last_offset)
+      last_offset += m.length
+    end
+
+    # Hexadecimal password hashes
+    buffer.scan(
+      /[a-f0-9]{16,128}/mi
+    ).each do |m|
+      next unless m.length % 2 == 0
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      report_hit(:type => 'CommonHash', :data => m, :offset => source_offset + last_offset)
+      last_offset += m.length
+    end
+
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/pem.rb

@@ -0,0 +1,22 @@
+module Memorandom
+module Plugins
+class PEM < PluginTemplate
+
+  @@description = "This plugin looks for PEM-encoded data (keys, certificates, crls, etc)"
+  @@confidence  = 0.90
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+    buffer.scan(
+      /-----BEGIN\s*[^\-]+-----+\r?\n[^\-]*-----END\s*[^\-]+-----\r?\n?/m
+    ).each do |m|
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      report_hit(:plugin => self, :type => 'PEM', :data => m, :offset => source_offset + last_offset)
+      last_offset += m.length
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/rsa.rb

@@ -0,0 +1,71 @@
+module Memorandom
+module Plugins
+class RSA < PluginTemplate
+
+  require 'openssl'
+
+  @@description = "This plugin looks for RSA keys by finding Bignum-encoded p-values"
+  @@confidence  = 0.90
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+
+    key_lengths = [1024, 2048]
+
+    key_lengths.each do |bits|
+      p_size = bits / 16
+      n_size = bits / 8
+
+      found = []
+      0.upto(buffer.length - (p_size + n_size)) do |p_offset|
+
+        # Look for a prime of the right size (p or q)
+        found_p = OpenSSL::BN.new( buffer[p_offset, p_size].unpack("H*").first.to_i(16).to_s )
+        next unless found_p.prime?
+        next unless found_p > 0x1000000000000000000000
+
+        # Look for a modulus that matches the found p/q value
+        0.upto(buffer.length - (p_size + n_size) ) do |n_offset|
+
+          next if (n_offset < p_offset and (n_offset + n_size) > p_offset)
+          next if (p_offset < n_offset and (p_offset + p_size) > n_offset)
+
+          found_n = OpenSSL::BN.new(buffer[n_offset, n_size].unpack("H*").first.to_i(16).to_s )
+
+          next if found_n == 0
+          next if found_n == found_p
+          next unless found_n > 0x1000000000000000000000
+          next unless (found_n % found_p == 0)
+
+          found << [found_p, found_n, p_offset]
+        end
+      end
+
+      found = found.uniq 
+
+      next unless found.length > 0
+
+      mods = {}
+
+      # Track the last unique p/q value for a potential modulus
+      found.each do |info|
+        mods[ info[1] ] ||= {}
+        mods[ info[1] ][ info[0] ] = info[2]
+      end
+      p mods
+
+      next 
+      
+      mods.keys.each do |n|
+        uniq_pees = mods[n].keys.select do |k|
+          mods[n].keys.reject {|x| x == k}.select{|x| n == (x * k) }
+        end
+      end
+
+
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/plugins/template.rb

@@ -0,0 +1,46 @@
+module Memorandom
+class PluginTemplate
+
+  @@description = "This is an unconfigured plugin using the base template"
+  @@confidence  = 0.80
+
+  attr_accessor :scanner, :hits
+
+  def initialize(scanner)
+    self.scanner = scanner
+    self.hits = {}
+  end
+
+  def report_hit(info = {})
+    unless info[:offset] and info[:type]
+      raise RuntimeError, "No offset or type supplied in the hit: #{info.inspect}"
+    end
+
+    scanner.report_hit(info.merge({:plugin => self }))
+
+    offset = info.delete(:offset)
+    self.hits[offset] = info
+  end
+
+  def reset
+    self.hits = {}
+  end
+
+  def self.description
+    @@description
+  end
+
+  def self.confidence
+    @@confidence
+  end
+
+  def description
+    self.class.description
+  end
+
+  def confidence
+    self.class.confidence
+  end
+
+end
+end

data/lib/memorandom/plugins/url_params.rb

@@ -0,0 +1,22 @@
+module Memorandom
+module Plugins
+class URLParams < PluginTemplate
+
+  @@description = "This plugin looks for interesting URL parameters and POST data"
+  @@confidence  = 0.50
+
+  # Scan takes a buffer and an offset of where this buffer starts in the source
+  def scan(buffer, source_offset)
+    buffer.scan(
+      /[%a-z0-9_\-=\&]*(?:sid|session|sess|user|usr|login|pass|secret|token)[%a-z0-9_\-=\&]*=[%a-z0-9_\-=&]+/mi
+    ).each do |m|
+      # This may hit an earlier identical match, but thats ok
+      last_offset = buffer.index(m)
+      report_hit(:type => 'URLParams', :data => m, :offset => source_offset + last_offset)
+      last_offset += m.length
+    end
+  end
+
+end
+end
+end

data/lib/memorandom/scanner.rb

@@ -0,0 +1,152 @@
+module Memorandom
+  class Scanner
+
+    require 'fileutils'
+    require 'openssl'
+    require 'yaml'
+
+    attr_accessor :plugins, :source
+    attr_accessor :window, :overlap, :output
+
+    def initialize(opts={})
+      plugin_names = Memorandom::PluginManager.plugins.keys
+
+      # Allow only a subset of plugins to be selected
+      if opts[:plugins]
+        plugin_names = Memorandom::PluginManager.plugins.keys.select{ |name|  
+          opts[:plugins].include?(name) 
+        }
+      end
+
+      # Load the selected plugins
+      self.plugins = plugin_names.map { |name|
+        Memorandom::PluginManager.plugins[name].new(self)
+      }
+
+      self.window  = opts[:window]  || 1024*1024
+      self.overlap = opts[:overlap] || 1024*4
+      self.output  = opts[:output]
+
+      FileUtils.mkdir_p(self.output) if self.output
+
+    end
+
+    def scan(target, source_name = nil)
+      fd = nil
+
+      if target.respond_to?(:read)
+        self.source = source_name || target.to_s
+      else
+        case target
+        when '-'
+          fd = $stdin
+          self.source = source_name || '<stdin>'
+        else
+          unless ( File.file?(target) or File.blockdev?(target) or File.chardev?(target) )
+            display("[-] Skipping #{target}: not a file")
+            return
+          end
+          begin
+            fd = File.open(target, "rb")
+            self.source = source_name ||"file:#{target.dup}"
+          rescue ::Interrupt
+            raise $!
+          rescue ::Exception
+            display("[-] Skipping #{target}: #{$!.class} #{$!}")
+            return
+          end
+        end
+      end
+
+      # Reset the plugin state between each target
+      self.plugins.each { |plugin| plugin.reset }
+
+      buffer = fd.read(self.window)
+      offset = 0
+
+      # Skip empty sources (an empty first read)
+      return unless buffer
+
+      while buffer.length > 0
+
+        self.plugins.each do |plugin|
+          # display("[*] Scanning #{buffer.length} bytes at offset #{offset}")
+
+          # Track a temporary per-plugin buffer and offset
+          scan_buffer = buffer
+          scan_offset = offset 
+
+          # Adjust the buffer and offset if any hits have been found that are 
+          # greater than offset. This is required because of overlap between 
+          # search windows. It is rare, but should be handled here so that
+          # plugins don't have to worry about it.
+
+          adjust = plugin.hits.keys.select{|hit| hit > offset }.last
+          if adjust
+            start_index = ( adjust - offset + 1 )
+            scan_buffer = buffer[start_index, buffer.length-start_index]
+            scan_offset += start_index
+          end
+
+          # Scan the buffer with the plugin
+          plugin.scan(scan_buffer, scan_offset)
+        end
+
+        # Calculate the next sliding window 
+        seeked = self.window - self.overlap
+        offset += seeked
+
+        nbytes = fd.read(seeked)
+        break if not nbytes
+
+        # Append new data to the end of the buffer
+        buffer << nbytes
+
+        # Delete most of the previous data from the beginning
+        buffer[0, seeked] = ''
+      end
+
+      # Close the file descriptor if we opened it
+      fd.close if source =~ /^file:/
+    end
+
+    def report_hit(info)
+      unless self.output
+        display("[+] #{source} #{info[:type]}@#{info[:offset]} (#{info[:data][0,32].inspect}...)")
+        return
+      end
+
+      fname = source.split("/").last[0,128] + "_"
+      fname << info[:data][0,16].unpack("H*").first
+      fname << "_#{info[:offset]}.#{info[:type]}"
+      yname = fname + ".yml"
+
+      fname = clean_filename(fname)
+      yname = clean_filename(yname)
+
+      fpath = ::File.join(self.output, fname)
+      ::File.open(fpath, "wb") { |fd| fd.write(info[:data]) }
+
+      display("[+] #{source} #{info[:type]}@#{info[:offset]} (#{info[:data][0,32].inspect}) stored in #{fpath}")
+
+      ypath = ::File.join(self.output, yname)
+      yhash = { 
+        :source    => source, 
+        :type      => info[:type], 
+        :offset    => info[:offset], 
+        :timestamp => Time.now,
+        :length    => info[:data].length
+      }
+      ::File.open(ypath, "wb") { |fd| fd.write(yhash.to_yaml) }
+
+    end
+
+    def clean_filename(name)
+      name.gsub(/[^a-zA-Z0-9_\.\-]/, '_').gsub(/_+/, '_')
+    end
+
+    def display(str)
+      $stdout.puts(str)
+    end
+  end
+end

data/lib/memorandom/version.rb

@@ -0,0 +1,3 @@
+module Memorandom
+  VERSION = "0.0.1"
+end

data/memorandom.gemspec

@@ -0,0 +1,34 @@
+# -*- encoding: utf-8 -*-
+$LOAD_PATH.push File.expand_path('../lib', __FILE__)
+require 'memorandom/version'
+
+Gem::Specification.new do |s|
+  s.name        = 'memorandom'
+  s.version     = Memorandom::VERSION
+  s.authors     = [
+      'Rapid7 Research'
+  ]
+  s.email       = [
+      'research@rapid7.com'
+  ]
+  s.homepage    = "https://www.github.com/rapid7/memorandom"
+  s.summary     = %q{Utility and library for extracting secrets from binary blobs}
+  s.description = %q{
+    Memorandom provides a command-line utility and class library from extracting secrets
+    from binary files. Common use cases include extracting encryption keys from memory dumps
+    and identifying sensitive data stored in block devices.
+  }.gsub(/\s+/, ' ').strip
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+
+  # ---- Dependencies ----
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'cucumber' 
+  s.add_development_dependency 'aruba'
+
+  s.add_runtime_dependency 'credit_card_validator'
+end

metadata

@@ -0,0 +1,130 @@
+--- !ruby/object:Gem::Specification
+name: memorandom
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Rapid7 Research
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-04-23 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: cucumber
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: aruba
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: credit_card_validator
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Memorandom provides a command-line utility and class library from extracting
+  secrets from binary files. Common use cases include extracting encryption keys from
+  memory dumps and identifying sensitive data stored in block devices.
+email:
+- research@rapid7.com
+executables:
+- memorandom.rb
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- LICENSE
+- README.md
+- bin/memorandom.rb
+- lib/memorandom.rb
+- lib/memorandom/plugins.rb
+- lib/memorandom/plugins/aes.rb
+- lib/memorandom/plugins/capi.rb
+- lib/memorandom/plugins/cc.rb
+- lib/memorandom/plugins/der.rb
+- lib/memorandom/plugins/hashes.rb
+- lib/memorandom/plugins/pem.rb
+- lib/memorandom/plugins/rsa.rb
+- lib/memorandom/plugins/template.rb
+- lib/memorandom/plugins/url_params.rb
+- lib/memorandom/scanner.rb
+- lib/memorandom/version.rb
+- memorandom.gemspec
+homepage: https://www.github.com/rapid7/memorandom
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Utility and library for extracting secrets from binary blobs
+test_files: []