crypto-toolbox 0.1.9 → 0.1.10

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 24485a636d5cc38818c7a94973eb2315c6f99a6c
-  data.tar.gz: 7d6a52bb6e920fbf76cafc6e69c2945c62a67433
+  metadata.gz: 2eab51c42b84900135dc165a677a18be4d470a1b
+  data.tar.gz: a2ec0819114133ae5f613ef2ce4ed153c10a3dff
 SHA512:
-  metadata.gz: 251989795e3c562c979523a2c2dac33f51f74a7cba13817bf1a8ed5e695838b3220f02b1a237405276c01a248852cf0bdadc137ebfc940ded46c3d20a9269dd9
-  data.tar.gz: 6e9a96b977eb116e2d2e0f4026074ac08565d250e0ceb9b36516a8ee00257e3cdf84fac99fabfca872f6c9b9c05c8105bba874220ba834fdc4301ea1b531fbcd
+  metadata.gz: 340f89d19a2bb483beffa8b93c16c4c260c6ddb289ce2e187fb53b731413078eb7aa79b8c93b633ad1d5e8e29bde23eded39b7605d873530c0b30c3aff50aaa8
+  data.tar.gz: 0be69d404f8007cd889051c403e92ab4cf09b0600bb8aa04432115c3042fadcb1505429f873cfbd7625333206826cd12a7ac5bcc036eeedb7c3e2626e1fe5c60

data/bin/break-vigenere-xor

@@ -9,13 +9,3 @@ else
 
   Analyzers::VigenereXor.new.analyze(ciphertext)
 end
-
-
-
-
-
-
-
-
-
-

data/lib/crypto-toolbox.rb

@@ -1,10 +1,12 @@
+require 'crypto-toolbox/crypt_buffer_input_converter.rb'
 require 'crypto-toolbox/crypt_buffer.rb'
 
-require 'crypto-toolbox/key_filter.rb'
-require 'crypto-toolbox/spell_checker.rb'
 
+require 'crypto-toolbox/analyzers/utils/key_filter.rb'
+require 'crypto-toolbox/analyzers/utils/spell_checker.rb'
 require 'crypto-toolbox/analyzers/padding_oracle.rb'
 require 'crypto-toolbox/analyzers/vigenere_xor.rb'
 
+
 require 'crypto-toolbox/ciphers/caesar.rb'
 require 'crypto-toolbox/ciphers/rot13.rb'

data/lib/crypto-toolbox/analyzers/utils/key_filter.rb

@@ -0,0 +1,44 @@
+require 'crypto-toolbox/analyzers/utils/spell_checker.rb'
+
+module Analyzers
+  module Utils
+    module KeyFilter
+      class AsciiPlain
+
+        
+        def initialize(keys,ciphertext,dict_lang="en_GB")
+          @keys = keys
+          @c = @ciphertext = ciphertext
+          @keylen = keys.first.length
+          @dict = FFI::Hunspell.dict(dict_lang)
+        end
+
+        def filter
+          # how often is the key repeated 
+          reps = @c.bytes.length / @keylen
+          result =[]
+          spell_checker = Analyzers::Utils::SpellChecker.new("en_GB")
+
+          # should we fork here ?
+          @keys.each_with_index do |key,i| #  i is used as a simple counter only !
+            test = CryptBuffer.new(@c.bytes[0,@keylen]).xor(key).str
+            repkey = CryptBuffer.new((key*reps) + key[0,(@c.bytes.length % reps).to_i])
+            str    = @c.xor(repkey).to_s
+
+            if spell_checker.human_language?(str)
+              result << repkey
+              break
+            else
+              if (i % 50000).zero?
+                puts "[Progress] #{i}/#{@keys.length} (#{(i.to_f/@keys.length*100).round(4)}%)"
+              end
+            end
+          end
+          return result
+        end
+
+        
+      end
+    end
+  end
+end

data/lib/crypto-toolbox/analyzers/utils/spell_checker.rb

@@ -0,0 +1,57 @@
+require 'ffi/hunspell'
+
+
+module Analyzers
+  module Utils
+    class SpellChecker
+
+      
+      def initialize(dict_lang="en_GB")
+        @dict = FFI::Hunspell.dict(dict_lang)
+      end
+=begin
+NOTE: About spelling error rates and language detection:
+
+missing punctuation support may lead to > 2% errors on valid texts, thus we use a high value .
+invalid decryptions tend to have spell error rates > 70
+Some statistics about it:
+> summary(invalids)
+   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
+ 0.6000  1.0000  1.0000  0.9878  1.0000  1.0000 
+> summary(cut(invalids,10))
+ (0.6,0.64] (0.64,0.68] (0.68,0.72] (0.72,0.76]  (0.76,0.8]  (0.8,0.84] 
+          8          13           9         534        1319        2809 
+(0.84,0.88] (0.88,0.92] (0.92,0.96]    (0.96,1] 
+      10581       46598      198477     1440651 
+=end
+      def known_words(str)
+        words = str.split(" ").select{|w| @dict.check?(w) }
+      end
+
+      def suggest(str)
+        @dict.suggest(str)
+      end
+
+      # Check whether a given string seems to be part of a human language using the given dictionary
+      #
+      # NOTE:
+      # Using shell instead of hunspell ffi causes lots of escaping errors, even with shellwords.escape
+      # errors = Float(`echo '#{Shellwords.escape(str)}' |hunspell -l |wc -l `.split.first)
+      def human_language?(str)
+        words  = str.split(" ").length
+        errors = str.split(" ").map{|e| @dict.check?(e) }.count{|e| e == false}
+        
+        error_rate = errors.to_f/words
+        
+        $stderr.puts error_rate.round(4) if ENV["CRYPTO_TOOBOX_PRINT_ERROR_RATES"]
+        
+        error_rate_sufficient?(error_rate)
+      end
+      
+      private
+      def error_rate_sufficient?(rate)
+        rate < 0.5
+      end
+    end
+  end
+end

data/lib/crypto-toolbox/analyzers/vigenere_xor.rb

@@ -8,6 +8,25 @@
 =end
 module Analyzers
   class VigenereXor
+    # This crypto analyzers takes a hex encoded ciphertext as input string
+    # and tries to find the plaintext by doing the following crypto analysis:
+    #
+    # 1) Search for a recurring pattern of the 8th bit of the ciphertext
+    # since ascii plaintext chars to have this bit set, the pattern will
+    # imply the key length
+    #
+    # 2) Create a map of all possible bytes for every position of the key
+    # The amount of candidates can be reduced by only allowing bytes that
+    # lead to a ascii english char
+    #
+    # 3) create the product of all possible combinations
+    # This only works for short key lengths due to the exponential growth
+    #
+    # 4) Do an English language Analysis of the possible result by using
+    # the error rate of the candidate plaintext using hunspell
+    #
+
+    
     def jot(message, debug: false)
       if debug == false || ENV["DEBUG_ANALYSIS"]
         puts message
@@ -25,7 +44,7 @@ module Analyzers
     def find_pattern(buf)
       bitstring = buf.nth_bits(7).join("")
       
-      1.upto([buf.bytes.length,62].min).map do |ksize|
+      1.upto(buf.bytes.length).map do |ksize|
         parts = bitstring.scan(/.{#{ksize}}/)
         if parts.uniq.length == 1
           parts.first
@@ -34,68 +53,77 @@ module Analyzers
         end
       end.compact.first
     end
-    
-    def analyze(input)
-      buf = CryptBuffer.from_hex(input)
-      result = find_pattern(buf)
-
-      if result.nil?
-        $stderr.puts "failed to find keylength by ASCII-8-Bit anlysis"
-        exit(1)
-      end
-
-      keylen = result.length
-      jot "Found recurring key pattern: #{result}"
-      jot "Detected key length: #{keylen}"
 
+    def create_candidate_map(buf,keylen)
       candidate_map ={}
-      (0..(keylen-1)).each do |key_byte|
+      (0..(keylen-1)).each do |key_byte_pos|
 
-        nth_stream = (key_byte).step(buf.bytes.length() -1, keylen).map{|i| buf.bytes[i]}
+        nth_stream = (key_byte_pos).step(buf.bytes.length() -1, keylen).map{|i| buf.bytes[i]}
         smart_buf = CryptBuffer.new(nth_stream)
 
-        candidate_map[key_byte]=[]
-        1.upto(255).each do |possible_key_value|
-          if smart_buf.xor_all_with(possible_key_value).bytes.all?{|byte| acceptable_char?(byte) }
-            jot("YES: " + smart_buf.xor_all_with(possible_key_value).to_s,debug: true)
-            candidate_map[key_byte] << possible_key_value
+        candidate_map[key_byte_pos]=[]
+        1.upto(255).each do |guess|
+          if smart_buf.xor_all_with(guess).bytes.all?{|byte| acceptable_char?(byte) }
+            jot("YES: " + smart_buf.xor_all_with(guess).to_s,debug: true)
+            candidate_map[key_byte_pos] << guess
           else
             # the current byte does not create a plain ascii result ( thus skip it )
-            #jot  "NO: " + smart_buf.xor_all_with(possible_key_value).to_s
+            #jot  "NO: " + smart_buf.xor_all_with(guess).to_s
           end
         end
       end
+      
+      candidate_map
+    end
+    
+    def analyze(input)
+      buf = CryptBuffer.from_hex(input)
 
-      head,*tail = candidate_map.map{|k,v|v}
-
-      jot "Amount of candidate keys: #{candidate_map.map{|k,v| v.length}.reduce(&:*)}. Starting Permutation (RAM intensive)"  
+      # Example: "100100" || nil
+      key_pattern = find_pattern(buf)
+      if key_pattern.nil?
+        $stderr.puts "failed to find keylength by ASCII-8-Bit anlysis"
+        exit(1)
+      end
+      keylen = key_pattern.length
+      jot "Found recurring key pattern: #{key_pattern}"
+      jot "Detected key length: #{keylen}"
 
+      
+      candidate_map = create_candidate_map(buf,keylen)
+      jot "Amount of candidate keys: #{candidate_map.map{|k,v| v.length}.reduce(&:*)}. Starting Permutation (RAM intensive)"
+      
+      # split the candidate map into head and*tail to create the prduct of all combinations
+      head,*tail = candidate_map.map{|k,v|v}
       combinations = head.product(*tail)
-      # make sure all permutations are still according to the bytes per position map
-      #x = combinations.select do |arr|
-      #  #binding.pry
-      #  arr.map.with_index{|e,i| candidate_map[i].include?(e)  }.all?{|e| e ==true}
-      #end
-      if ENV["SEMI_AUTO_ANALYSIS"] && ENV["DEBUG_ANALYSIS"]
-        print_candidate_encryptions(candidate_map,keylen,buf)
+
+      if ENV["DEBUG_ANALYSIS"]
+        ensure_consistent_result!(combinations,candidate_map)
+        print_candidate_decryptions(candidate_map,keylen,buf)
       end
       
-      results = KeySearch::Filter::AsciiPlain.new(combinations,buf).filter
+      results = Analyzers::Utils::KeyFilter::AsciiPlain.new(combinations,buf).filter
       report_result(results,buf)
     end
 
+    def ensure_consistent_result!(combinations,condidate_map)
+      # NOTE Consistency check ( enable if you dont trust the generation anymore )
+      # make sure all permutations are still according to the bytes per position map
+      combinations.select do |arr|
+        raise "Inconsistent key candidate combinations" unless arr.map.with_index{|e,i| candidate_map[i].include?(e)  }.all?{|e| e ==true}
+      end      
+    end
+
     def report_result(results,buf)
        unless results.empty?
-        jot "[Success] Found valid result(s)"
+        jot "[Success] Found valid result(s):"
         results.each do |r|
-          print_delimiter_line
           jot r.xor(buf).str
-          print_delimiter_line
         end
       end
     end
     
-    def print_candidate_encryptions(candidate_map,keylen,buf)
+    def print_candidate_decryptions(candidate_map,keylen,buf)
       # printout for debugging. (Manual analysis of the characters)
       print "======= Decryption result of first #{keylen} bytes with all candidate keys =======\n" 
       (0..keylen-1).each do|i|
@@ -110,12 +138,6 @@ module Analyzers
   end
 end
 
-=begin
-NOTE: we may at digram and trigram support?
-#trigram="the "
-#x = CryptBuffer.new(trigram)
-=end
-
 
 
 

data/lib/crypto-toolbox/ciphers/caesar.rb

@@ -32,7 +32,7 @@ Letter Array include?(A):    76997.0 i/s - 42.73x slower
         mod    = (char =~ /[a-z]/) ? 123 : 91
         offset = (char =~ /[a-z]/) ? 97  : 65
         
-        (char =~ /[^a-zA-Z]/) ? char : CryptBuffer.new(char).add(real_shift, mod: mod, offset: offset).str
+        (char =~ /[^a-zA-Z]/) ? char : CryptBuffer(char).add(real_shift, mod: mod, offset: offset).str
       end.join
     end
 

data/lib/crypto-toolbox/crypt_buffer.rb

@@ -32,22 +32,16 @@ class CryptBuffer
   attr_accessor :bytes
   alias_method :b, :bytes
 
-
-
   
-  def initialize(input)
-    @bytes = bytes_from_any(input)
+  def initialize(byte_array)
+    @bytes = byte_array
   end
 
   # Make sure input strings are always interpreted as hex strings
   # This is especially useful for unknown or uncertain inputs like
   # strings with or without leading 0x
   def self.from_hex(input)
-    hexstr =""
-    unless input.nil?
-      hexstr = (input =~ /^0x/ ? input : "0x#{pad_hex_char(input)}" )
-    end
-    CryptBuffer.new(hexstr)
+    CryptBufferInputConverter.new.from_hex(input)
   end
 
   # Returns an array of the nth least sigificant by bit of each byte
@@ -61,68 +55,11 @@ class CryptBuffer
   def chunks_of(n)
     self.bytes.each_slice(n).map{|chunk| CryptBuffer(chunk) }
   end
-
-
-
   
   private
   def xor_multiple(byte,bytes)
     ([byte] + bytes).reduce(:^)
   end
-  
-  def bytes_from_any(input)
-    case input
-    when Array
-      input
-    when String
-      str2bytes(input)
-    when CryptBuffer
-      input.b
-    when Fixnum
-      int2bytes(input)
-    else
-      raise "Unsupported input: #{input.inspect} of class #{input.class}"
-    end
-  end
-
-  def normalize_hex(str)
-    tmp = self.class.pad_hex_char(str)
-    tmp.gsub(/(^0x|\s)/,"").upcase
-  end
-  
-  def self.pad_hex_char(str)
-    (str.length == 1) ? "0#{str}" : "#{str}"
-  end
-  
-  def strip_hex_prefix(hex)
-    raise "remove 0x from hexinput"
-  end
-  
-  def int2bytes(input)
-    # integers as strings dont have a 0x prefix
-    if input.to_s(16).match(/^[0-9a-fA-F]+$/)
-      # assume 0x prefixed integer
-      hex2bytes(normalize_hex(input.to_s(16)))
-    else
-      # regular number
-      [input].pack('C*').bytes
-    end
-  end
-  
-  def hex2bytes(hexstr)
-    hexstr.scan(/../).map{|h| h.to_i(16) }
-  end
-
-  def str2bytes(str)
-    if str.match(/^0x[0-9a-fA-F]+$/).nil?
-      str.bytes.to_a
-    else
-      hex2bytes(normalize_hex(str))
-    end
-  end
 end
 
 
-def CryptBuffer(input)
-  CryptBuffer.new(input)
-end

data/lib/crypto-toolbox/crypt_buffer/concerns/comparable.rb

@@ -1,7 +1,7 @@
 module CryptBufferConcern
   module Comparable
     def ==(other)
-      bytes == bytes_from_any(other)
+      bytes == CryptBuffer(other).bytes
     end
   end
 end

data/lib/crypto-toolbox/crypt_buffer/concerns/xor.rb

@@ -20,12 +20,12 @@ module CryptBufferConcern
       if expand_input
         xor_all_with(input)
       else
-        xor_bytes(bytes_from_any(input))
+        xor_bytes(CryptBuffer(input).bytes)
       end
     end
 
     def xor_all_with(input)
-      expanded = expand_bytes(bytes_from_any(input),self.bytes.length)
+      expanded = expand_bytes(CryptBuffer(input).bytes,self.bytes.length)
       xor_bytes(expanded)
     end
 

data/lib/crypto-toolbox/crypt_buffer_input_converter.rb

@@ -0,0 +1,72 @@
+
+
+class CryptBufferInputConverter
+  def convert(input)
+    bytes_from_any(input)
+  end
+
+  # Make sure input strings are always interpreted as hex strings
+  # This is especially useful for unknown or uncertain inputs like
+  # strings with or without leading 0x
+  def from_hex(input)
+    hexstr =""
+    unless input.nil?
+      hexstr = normalize_hex(input)
+    end
+    CryptBuffer.new(hex2bytes(hexstr))
+  end
+  
+  private
+  def bytes_from_any(input)
+    case input
+    when Array
+      input
+    when String
+      str2bytes(input)
+    when CryptBuffer
+      input.b
+    when Fixnum
+      int2bytes(input)
+    else
+      raise "Unsupported input: #{input.inspect} of class #{input.class}"
+    end
+  end
+
+  def int2bytes(input)
+    # integers as strings dont have a 0x prefix
+    if input.to_s(16).match(/^[0-9a-fA-F]+$/)
+      # assume 0x prefixed integer
+      hex2bytes(normalize_hex(input.to_s(16)))
+    else
+      # regular number
+      [input].pack('C*').bytes
+    end
+  end
+  
+  def hex2bytes(hexstr)
+    hexstr.scan(/../).map{|h| h.to_i(16) }
+  end
+
+  def str2bytes(str)
+    if str.match(/^0x[0-9a-fA-F]+$/).nil?
+      str.bytes.to_a
+    else
+      hex2bytes(normalize_hex(str))
+    end
+  end
+  
+  def pad_hex_char(str)
+    (str.length == 1) ? "0#{str}" : "#{str}"
+  end
+  
+  def normalize_hex(str)
+    tmp = pad_hex_char(str)
+    tmp.gsub(/(^0x|\s)/,"").upcase
+  end
+
+end
+
+def CryptBuffer(input)
+  bytes = CryptBufferInputConverter.new.convert(input)
+  CryptBuffer.new(bytes)
+end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: crypto-toolbox
 version: !ruby/object:Gem::Version
-  version: 0.1.9
+  version: 0.1.10
 platform: ruby
 authors:
 - Dennis Sivia
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-04-21 00:00:00.000000000 Z
+date: 2015-04-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: aes
@@ -52,6 +52,8 @@ files:
 - lib/crypto-toolbox/analyzers/padding_oracle/analyzer.rb
 - lib/crypto-toolbox/analyzers/padding_oracle/oracles/http_oracle.rb
 - lib/crypto-toolbox/analyzers/padding_oracle/oracles/tcp_oracle.rb
+- lib/crypto-toolbox/analyzers/utils/key_filter.rb
+- lib/crypto-toolbox/analyzers/utils/spell_checker.rb
 - lib/crypto-toolbox/analyzers/vigenere_xor.rb
 - lib/crypto-toolbox/ciphers/caesar.rb
 - lib/crypto-toolbox/ciphers/rot13.rb
@@ -64,8 +66,7 @@ files:
 - lib/crypto-toolbox/crypt_buffer/concerns/pretty_print.rb
 - lib/crypto-toolbox/crypt_buffer/concerns/random.rb
 - lib/crypto-toolbox/crypt_buffer/concerns/xor.rb
-- lib/crypto-toolbox/key_filter.rb
-- lib/crypto-toolbox/spell_checker.rb
+- lib/crypto-toolbox/crypt_buffer_input_converter.rb
 homepage: https://github.com/scepticulous/crypto-toolbox
 licenses:
 - GPLv3

data/lib/crypto-toolbox/key_filter.rb

@@ -1,40 +0,0 @@
-require_relative './crypt_buffer.rb'
-require_relative './spell_checker.rb'
-
-module KeySearch
-  module Filter
-    class AsciiPlain
-      def initialize(keys,ciphertext,dict_lang="en_GB")
-        @keys = keys
-        @c = @ciphertext = ciphertext
-        @keylen = keys.first.length
-        @dict = FFI::Hunspell.dict(dict_lang)
-      end
-
-
-      def filter
-        # how often is the key repeated 
-        reps = @c.bytes.length / @keylen
-        result =[]
-        spell_checker = SpellChecker.new("en_GB")
-        
-        @keys.each_with_index do |key,i| #  i is used as a simple counter only !
-          test = CryptBuffer.new(@c.bytes[0,@keylen]).xor(key).str
-          repkey = CryptBuffer.new((key*reps) + key[0,(@c.bytes.length % reps).to_i])
-          str    = @c.xor(repkey).to_s
-
-          if spell_checker.human_language?(str)
-            result << repkey
-            break
-          else
-            if (i % 50000).zero?
-              puts "[Progress] #{i}/#{@keys.length} (#{(i.to_f/@keys.length*100).round(4)}%)"
-            end
-          end
-        end
-        return result
-      end
-      
-    end
-  end
-end

data/lib/crypto-toolbox/spell_checker.rb

@@ -1,48 +0,0 @@
-require 'ffi/hunspell'
-class SpellChecker
-  def initialize(dict_lang="en_GB")
-    @dict = FFI::Hunspell.dict(dict_lang)
-  end
-=begin
-NOTE: About spelling error rates and language detection:
-
-missing punctuation support may lead to > 2% errors on valid texts, thus we use a high value .
-invalid decryptions tend to have spell error rates > 70
-Some statistics about it:
-> summary(invalids)
-   Min. 1st Qu.  Median    Mean 3rd Qu.    Max. 
- 0.6000  1.0000  1.0000  0.9878  1.0000  1.0000 
-> summary(cut(invalids,10))
- (0.6,0.64] (0.64,0.68] (0.68,0.72] (0.72,0.76]  (0.76,0.8]  (0.8,0.84] 
-          8          13           9         534        1319        2809 
-(0.84,0.88] (0.88,0.92] (0.92,0.96]    (0.96,1] 
-      10581       46598      198477     1440651 
-=end
-  def known_words(str)
-    words = str.split(" ").select{|w| @dict.check?(w) }
-  end
-
-  def suggest(str)
-    @dict.suggest(str)
-  end
-                                                        
-  def human_language?(str)
-    words  = str.split(" ").length
-    errors = str.split(" ").map{|e| @dict.check?(e) }.count{|e| e == false}
-    # using shell instead of hunspell ffi causes lots of escaping errors, even with shellwords.escape
-    #errors = Float(`echo '#{Shellwords.escape(str)}' |hunspell -l |wc -l `.split.first)
-
-    error_rate = errors.to_f/words
-          
-    $stderr.puts error_rate.round(4) if ENV["CRYPTO_TOOBOX_PRINT_ERROR_RATES"]
-
-    if error_rate < 0.5
-      puts "[Success] Found valid result (spell error_rate: #{error_rate*100}% is below threshold: 20%)"
-      return true
-    else
-      return false
-    end
-  end
-
-    
-end