crypto-toolbox 0.2.3 → 0.2.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 0f9b9033bbd39aad71fa0a549e3af7e8b8c8b997
-  data.tar.gz: f9e59d8234a4acb4a123d7ca7e83f7cb638acae4
+  metadata.gz: 65cce12b1b52be22312dcdbe387a696e0b5e2dbc
+  data.tar.gz: 79e26f00e777f4f1fb76af853a171419f23aa1b2
 SHA512:
-  metadata.gz: ff3d96493b71d4a8cd6c4fa2b5f048cc5c3b04f0b03f51d14914173dd85587dd0f02e0f0e257917b54f19bcbd4380992fe2afd5a5b28cb0bbe4d144df7eb7faf
-  data.tar.gz: 8be45f26f5ca4ad275161df8166ff14727c4f1c5d5277a5ac4113ad1a9560bef5ceeb8ff45ab801f70a7da2e9568f65747007737c3d6d3785def1da6f792170a
+  metadata.gz: becd7a86fa057831193646cb161799c2f4dd85319990294b3c608b78c2b8cc774cefadd11d0306ac5b5c5f57cd82c419875f08b3cef36aacabd59bbf943b650f
+  data.tar.gz: 00c1bba37d1e37691ee0245f5f8f09004751479bb5002fabe5b42e8222a9449161b39ef025d74e428097e98fa9d25d4256367e1fb801924a7a21cc930e6d704b

data/bin/break-vigenere-xor

@@ -1,16 +1,19 @@
 #!/usr/bin/env ruby
 
 require 'crypto-toolbox'
+require 'stackprof'
 
 if ARGV[0].nil?
   $stderr.puts "Missing Argument: Ciphertext (hexstring)"
 else
   ciphertext = ARGV[0]
 
+StackProf.run(mode: :cpu, out: 'tmp/stackprof-cpu-vigenere-xor.dump') do
   results = Analyzers::VigenereXor.new.analyze(ciphertext)
   unless results.empty?
     puts "[Success] Found valid result(s):"
     puts results.map(&:str)
   end
+end
 
 end

data/lib/crypto-toolbox.rb

@@ -1,14 +1,17 @@
+# coding: utf-8
 require 'crypto-toolbox/utils/reporting/console.rb'
+require 'crypto-toolbox/utils/hamming_distance_filter.rb'
 
 require 'crypto-toolbox/crypt_buffer_input_converter.rb'
 require 'crypto-toolbox/crypt_buffer.rb'
 
 require 'crypto-toolbox/analyzers/utils/key_filter.rb'
-
+require 'crypto-toolbox/analyzers/utils/letter_frequency.rb'
 require 'crypto-toolbox/analyzers/utils/ascii_language_detector.rb'
 require 'crypto-toolbox/analyzers/utils/spell_checker.rb'
 require 'crypto-toolbox/analyzers/utils/human_language_detector.rb'
 
+
 require 'crypto-toolbox/analyzers/padding_oracle.rb'
 require 'crypto-toolbox/analyzers/cbc_mac.rb'
 require 'crypto-toolbox/analyzers/vigenere_xor.rb'
@@ -17,7 +20,6 @@ require 'crypto-toolbox/ciphers/aes.rb'
 require 'crypto-toolbox/ciphers/caesar.rb'
 require 'crypto-toolbox/ciphers/rot13.rb'
 
-
 require 'crypto-toolbox/forgers/stream_ciphers/forge_generator.rb'
 
 require 'crypto-toolbox/crypto_challanges/solver.rb'

data/lib/crypto-toolbox/analyzers/utils/key_candidate_map.rb

@@ -25,9 +25,6 @@ module Analyzers
       # 3) xor any possible byte value (guess) with all nth's bytes
       # 4) select those guesses that decipher the nth-byte stream to only english plain ascii chars
       def run(input_buf,keylen)
-        #return run2(input_buf,keylen)
-        detector = Analyzers::Utils::HumanLanguageDetector.new
-        
         candidate_map = (0..(keylen-1)).each_with_object({}) do |key_byte_pos,hsh|
 =begin
 # Letter frquency testing
@@ -41,11 +38,6 @@ module Analyzers
           # create an array of every nth byte of the input. ( thus a pseudo stream of the nth bytes )
           # 1) create an enumerator of the nth positions. e.g for iteration 0: [0,7,14,...]
           # 2) Next: Map the positions to bytes of the input buffer
-          #
-          # NOTE: regular implementation without cryptbuffer magic:
-          # nth_stream = (key_byte_pos).step(input_buf.bytes.length() -1, keylen).map{|i| input_buf.bytes[i]}
-          # nth_byte_stream2 = CryptBuffer.new(nth_stream)
-
           nth_byte_stream = input_buf.nth_bytes(keylen,offset: key_byte_pos)
           hsh[key_byte_pos] = 0.upto(255).select{|guess| nth_byte_stream.xor_all_with(guess).bytes.all?{|byte| acceptable_char?(byte) } }
           

data/lib/crypto-toolbox/analyzers/utils/key_filter.rb

@@ -5,27 +5,31 @@ module Analyzers
     module KeyFilter
       class AsciiPlain
 
-        
-        def initialize(keys,ciphertext,dict_lang="en_US")
+        def initialize(keys,ciphertext)
           @keys = keys
           @c = @ciphertext = ciphertext
           @keylen = keys.first.length
-          @dict = FFI::Hunspell.dict(dict_lang)
+          @detector = Analyzers::Utils::HumanLanguageDetector.new
+          @spell_checker = Analyzers::Utils::SpellChecker.new("en_US")
         end
 
         def filter
           # how often is the key repeated 
           reps = @c.bytes.length / @keylen
           result =[]
-          spell_checker = Analyzers::Utils::SpellChecker.new("en_US")
 
+          
+
+          
           # should we fork here ?
           @keys.each_with_index do |key,i| #  i is used as a simple counter only !
             test = CryptBuffer.new(@c.bytes[0,@keylen]).xor(key).str
             repkey = CryptBuffer.new((key*reps) + key[0,(@c.bytes.length % reps).to_i])
-            str    = @c.xor(repkey).to_s
-
-            if spell_checker.human_language?(str)
+            str  = @c.xor(repkey).to_s
+            
+            # NOTE: we dont need the ASCII check provided by the human language detector
+            # since the key selection is usually based on ascii value checks
+            if @spell_checker.human_language?(str)
               result << repkey
               break
             else

data/lib/crypto-toolbox/analyzers/utils/letter_frequency.rb

@@ -18,19 +18,37 @@ module Analyzers
         'u' =>	2.88,
         'c' =>	2.71
       }
+
+      
       def letter_count(str)
         str.downcase.each_char.with_object({}) do |c,h|
-          h[c] = (h.fetch(c,0) + 1) if c =~ /[A-Za-z ]/
+          h[c] = increment_letter_count(h,c) if countable?(c)
         end
       end
       
       def letter_freq(str)
-        counts   = letter_count(str)
-        quotient = counts.values.reduce(&:+).to_f
-        counts.sort_by{|k,v| v}.reverse.to_h.each_with_object({}){|(k,v),hsh| hsh[k] = (v/quotient) }
+        counts      = letter_count(str)
+        total_chars = counts.values.reduce(&:+)
+        Hash[reverse_hash(counts).map{|k,v| [k,calculate_frequency(v,total_chars)] } ]
       end
 
+      
+      private
 
+      def reverse_hash(hsh)
+        hsh.sort_by{|k,v| -v}
+      end
+      def calculate_frequency(value,total)
+        (value/total.to_f).round(4)
+      end
+
+      def increment_letter_count(hsh,char)
+        (hsh.fetch(char,0) + 1) 
+      end
+
+      def countable?(char)
+        char =~ /[A-Za-z ]/
+      end
     end
   end
 end

data/lib/crypto-toolbox/analyzers/utils/spell_checker.rb

@@ -1,4 +1,5 @@
 require 'ffi/hunspell'
+require 'ffi/aspell'
 
 module Analyzers
   module Utils
@@ -6,6 +7,7 @@ module Analyzers
       
       def initialize(dict_lang="en_US")
         @dict = FFI::Hunspell.dict(dict_lang)
+        @dict2 = FFI::Aspell::Speller.new(dict_lang)
       end
 =begin
 NOTE: About spelling error rates and language detection:
@@ -69,8 +71,13 @@ if numbers or single char words are taken into account
         end
       end
 
+      # note:
+      # Aspell is much faster but requires expensive and slow removal of all punctuation marks
+      # which makes it slower than hunspell.
+      # Thus we stick with hunspell for correctness and speed.
       def check?(input)
-        @dict.check?(input) rescue false 
+        @dict.check?(input)
+        # @dict2.correct?(input.gsub(/[^a-zA-Z]/,""))
       end
       
       def error_rate_sufficient?(rate)

data/lib/crypto-toolbox/analyzers/vigenere_xor.rb

@@ -30,14 +30,7 @@ module Analyzers
 
     class HammingDistanceKeyLengthFinder
       def keylen_for(buffer)
-        offset = 2
-        distances = ((0+offset)..64).map do |keysize|
-          # take the first 4 blocks of keysize length, generate all combinations (6),
-          # map than to normalized hamming distance and take mean
-          buffer.chunks_of(keysize)[0,4].combination(2).map{|a,b| a.hdist(b,normalize: true)}.reduce(&:+) / 6.0
-        end
-        # get the min distance, find its index, convert the keylen
-        distances.min(4).map{|m| distances.index(m)}.map{|i| i + offset }.uniq
+        ::Utils::HammingDistanceFilter.new.shortest_distance_entries(buffer)
       end
     end
     

data/lib/crypto-toolbox/crypt_buffer/concerns/convertable.rb

@@ -7,12 +7,12 @@ module CryptBufferConcern
     alias_method :h, :hex
     
     def chars
-      map{|b| b.to_i.chr}
+      map(&:chr)
     end
     alias_method :c, :chars
     
     def str
-      chars.join
+      map(&:chr).join
     end
     alias_method :s, :str
 

data/lib/crypto-toolbox/crypto_challanges/solver.rb

@@ -8,18 +8,6 @@ module CryptoChallanges
     def solve2(c1,c2)
       (CryptBuffer.from_hex(c1) ^ CryptBuffer.from_hex(c2)).hex.downcase
     end
-
-    def letter_count(str)
-      str.downcase.each_char.with_object({}) do |c,h|
-        h[c] = (h.fetch(c,0) + 1) if c =~ /[A-Za-z ]/
-      end
-    end
-    
-    def letter_freq(str)
-      counts   = letter_count(str)
-      quotient = counts.values.reduce(&:+).to_f
-      counts.sort_by{|k,v| v}.reverse.to_h.each_with_object({}){|(k,v),hsh| hsh[k] = (v/quotient) }
-    end
     
     def solve3(input)
       candidates = (1..256).map{ |guess| CryptBuffer.from_hex(input).xor_all_with(guess) }
@@ -27,7 +15,9 @@ module CryptoChallanges
       
       detector.human_language_entries(candidates).first.to_s
     end
-    
+
+    # challange:
+    # One of the 60-character strings in this file has been encrypted by single-character XOR.
     def solve4(hexstrings)
       detector = Analyzers::Utils::HumanLanguageDetector.new
       result = hexstrings.map{|h| CryptBuffer.from_hex(h)}.map.with_index do |c,i|

data/lib/crypto-toolbox/utils/hamming_distance_filter.rb

@@ -0,0 +1,15 @@
+module Utils
+  class HammingDistanceFilter
+    def shortest_distance_entries(buffer,result_entries: 4,samples: 4)
+      offset = 2
+      distances = ((0+offset)..64).map do |keysize|
+        # take the first 4 blocks of keysize length, generate all combinations (6),
+        # map than to normalized hamming distance and take mean
+        buffer.chunks_of(keysize)[0,samples].combination(2).map{|a,b| a.hdist(b,normalize: true)}.reduce(&:+) / 6.0
+      end
+      # get the min distance, find its index, convert the keylen
+      distances.min(result_entries).map{|m| distances.index(m)}.map{|i| i + offset }.uniq
+    end
+  end
+end
+  

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: crypto-toolbox
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.4
 platform: ruby
 authors:
 - Dennis Sivia
@@ -82,6 +82,7 @@ files:
 - lib/crypto-toolbox/crypt_buffer_input_converter.rb
 - lib/crypto-toolbox/crypto_challanges/solver.rb
 - lib/crypto-toolbox/forgers/stream_ciphers/forge_generator.rb
+- lib/crypto-toolbox/utils/hamming_distance_filter.rb
 - lib/crypto-toolbox/utils/reporting/console.rb
 homepage: https://github.com/scepticulous/crypto-toolbox
 licenses: