heitt 0.4.5 → 0.4.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaae42753b6fd3c04375200d118517b8b3f45eb42d0bf6b4fddc9b08c64e5af7
-  data.tar.gz: ed0b83ef1116ebc291b000985784107a709156df68be16443598d2a41ae59853
+  metadata.gz: 2682eb074a7763036e54775706be2ae16221acd2d4cd7d416b387723e1b51aa2
+  data.tar.gz: 4739a64851b2e071217eed0b12e3e71327670a4abcdebd218fdb3020df4aa47a
 SHA512:
-  metadata.gz: c4f7ea072d396d4c5ba96f9f65402ab006c1a94856b77cd359417d7ddc26bde65bc81928369f64ad36562aa0d7ce4e7b0efbe4d04edb6d04e467c9df4ce5ab6d
-  data.tar.gz: 3a1c7494156de04f10c2a3f132f85337fdd597197883b81cc4d1df934fd3c9d70dcaf89378729837a7e0f086d6cb2d2c75518675e0af56b60e4ee7c3e2f1f804
+  metadata.gz: d6084258bca3bc35f40aecf9402dffbace9a419561f886e02a73939e821a7369798d4878b3e6483b4293687082ff169f7206c5046d94fc64ad38085718c758f5
+  data.tar.gz: 362c46e00ba6dde92e1763e39b2f965e7782fb46a2073ab25974bcbb9d9213bfa03719b30b6747a145e9d8fdf8bc200e26a9c8df88c8c1022b4657147ed13f6e

data/bin/heitt

@@ -2,6 +2,7 @@
 require 'optparse'
 require 'io/console'
 require 'heitt'
+#require_relative '../lib/heitt'
 
 
 
@@ -47,6 +48,7 @@ module HEITT
         opts.on("-V", "--verbose", "Show description and notes for each candidate") {@verbose = true}
         opts.on("-j", "--json","Output in json format") {@json = true}
         opts.on("-o", "--output FILEPATH", String, "File to write output to") {|v| @output = v}
+        opts.on("--debug", "Output in debug mode") {HEITT::Logger.enable_debug}
 
         opts.separator ""
         opts.separator "FILTERING OPTIONS:"
@@ -70,6 +72,7 @@ module HEITT
         opts.separator ""
         opts.separator "NOTES:"
         opts.separator "    JSON format is default when output is redirected or piped."
+        opts.separator "    Custom database if merged with default database"
         opts.separator "    Regex-match candidates are hidden by default, use '-r' to show."
         opts.separator "    Running without input starts interactive mode."
         opts.separator "#{header("=========================================================================")}"
@@ -79,9 +82,21 @@ module HEITT
 
       @inputs = if ARGV.empty?
         if $stdin.tty?
+          database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
           # Interactive mode
-          puts "#{header("=== HEITT INTERACTIVE MODE ====\n")}"
-          puts "Enter a hash or file"
+          puts header("=== HEITT INTERACTIVE MODE ====")
+          print "[debug:#{HEITT::Logger.debug_enabled? ? "on": "off"}  ||  "
+          print "extended: #{@extended ? "on": "off"}  ||  "
+          print "verbose: #{@verbose ? "on": "off"}  ||  "
+          print "regex-match: #{@show_regex_match ? "on": "off"}  ||  "
+          print "min-entropy: #{@min_entropy}  ||  "
+          print "database: #{@database.empty? ? "default": database}  ||  "
+          print "output-format: #{@json ? "json": "tree"}]\n"
+          puts ""
+          puts "Enter a hash or filepath. Press Ctrl+C or Enter to exit"
+          #puts "Extended: true" unless 
+          # puts used flags here
+          
           loop do
             print "heitt> "
             input = $stdin.gets&.strip
@@ -110,11 +125,16 @@ module HEITT
     end
 
     def run 
-      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)
+      HEITT::Logger.debug("Loading database...")
+      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
+      HEITT::Logger.debug("Database (#{@database.empty? ? "default" : @database}) loaded")
       
       format = @json || !$stdout.tty? ? :json : :tree
       output = @inputs.map do |input|
+        HEITT::Logger.debug("Scanning #{input}...")
         results = HEITT::Scanner.scan(input, database: database, min_entropy: @min_entropy)
+        HEITT::Logger.debug("Scanning completed (#{input})")
+        HEITT::Logger.debug("Grouping Results ...")
         groups = HEITT::Grouper.group(results)
 
         case format
@@ -139,16 +159,19 @@ module HEITT
     end
 
 
-    def load_custom_database(filepath)
+
+    def load_custom_database(file_path)
+      filepath = File.expand_path(file_path)
+      HEITT::Logger.debug("#{label} filepath: #{filepath}")
       unless File.exist?(filepath)
-        puts HEITT::Color.colorize("[ERROR] Database file #{filepath} does not exists", :bold, :red)
+        HEITT::Logger.error("#{label} file #{filepath} does not exist")
         exit(1)
       end
       begin
         custom_database = JSON.parse(File.read(filepath), symbolize_names: true)
         HEITT::DATABASE +  custom_database
       rescue JSON::ParserError => e 
-        puts "#{HEITT::Color.colorize("[ERROR] Invalid JSON in database file: ", :bold, :red)}#{e.message}"
+        HEITT::Logger.error("Invalid JSON in #{label} file: #{e.message}")
         exit(1)
       end
     end

data/lib/heitt/analyzer.rb

@@ -0,0 +1,156 @@
+require_relative 'utils'
+require_relative 'profiles'
+
+module HEITT
+  module Analyzer
+    def self.analyze(text, profiles: HEITT::PROFILES)#database: HEITT::DATABASE)
+      HEITT::Logger.debug("Counting keywords...")
+      keyword_counts = keyword_counts(text.downcase, profiles: profiles)
+      HEITT::Logger.debug("Counted keywords: #{keyword_counts}")
+      algorithm_scores(keyword_counts, profiles: profiles)
+    end
+
+    def self.extract_prefix(text, offset)
+      line_start = text.rindex("\n", offset) || 0
+      text[line_start...offset]
+    end
+
+    def self.high_entropy?(text, min_ent)
+      entropy(text) >= min_ent
+    end
+      
+
+    def self.score_candidates(modes, delim_prefix, context_scores, profiles: HEITT::PROFILES)
+      prefix_matched_mode = nil
+      #context based scoring
+      matches = modes.map do |mode|
+        profile = profiles[mode[:name]] || {}
+        score = context_scores[mode[:name]] || 0
+        #score = score_data || 0
+
+        if prefix_match?(profile, delim_prefix)
+          #boost score as confidence is high if prefix matched
+          prefix_matched_mode = mode[:name]
+          score += 20
+        end
+        {
+          name: mode[:name],
+          hashcat: mode[:hashcat],
+          john: mode[:john],
+          extended: mode[:extended],
+          description: profile[:description],
+          score: score
+        }
+      end 
+      return [] if matches.empty?
+
+      #calculate confidence
+      scores_hash = matches.map {|m| [m[:name], m[:score]]}.to_h 
+      
+      confidences = assign_confidence(scores_hash, prefix_matched_mode)
+      scored_candidates = matches.map{|m| m.merge(confidence: confidences[m[:name]])}.sort_by {|m| -m[:score]}
+      HEITT::Logger.debug("Scored Algorithm: #{scored_candidates.map{|s| s[:name] }}  =>   Calculated Confidence: #{scored_candidates.map{|s| s[:confidence] }}")
+      scored_candidates
+    end
+
+
+    private
+    #def self.get_modes(entry)
+    #  entry[:modes] || entry[:algorithms] || entry[:hashes] || 
+    #  entry[:candidates] || entry[:types] || entry[:hashtypes]
+    #end
+
+    #this  code is an inspiration of "https://github.com/chrisjchandler/entropy/blob/main/entropy.go"
+    def self.entropy(text)
+      frequency = Hash.new(0)
+      text.each_char { |ch| frequency[ch] += 1 }
+
+      #calculate the total number of characters
+      total = text.length.to_f
+      #caluclate entropy
+      entropy = 0.0
+      frequency.each_value do |count|
+        probability = count.to_f / total
+        entropy += probability * Math.log2(probability)
+      end
+      #negate the sum as entropy is positive
+      -entropy
+    end
+    
+    def self.keyword_counts(content_lower, profiles: HEITT::PROFILES) #HEITT::DATABASE)
+      keywords = profiles.values.flat_map { |p| p[:context] || [] }.uniq.map(&:downcase)
+      #database.flat_map do |entry|
+        #modes = get_modes(entry)
+        #next [] unless modes
+        #modes.flat_map {|mode| HEITT::PROFILES[mode[:name]][:context] || []}
+      #end#.uniq.map(&:downcase)
+
+      counts = {}
+      keywords.each do |kw|
+        count = content_lower.scan(/\b#{Regexp.escape(kw)}\b/).size
+        counts[kw] = count if count > 0
+      end
+      counts
+    end
+
+
+    def self.algorithm_scores(keyword_counts, profiles: HEITT::PROFILES)#database: HEITT::DATABASE)
+      scores = {}
+      return scores if keyword_counts.nil?
+
+      profiles.each do |name, profile| #database.each do |entry|
+        context = profile[:context] || []
+        next if context.empty?
+        #modes = get_modes(entry)
+        #next unless modes
+        #modes.each do |mode|
+        #  contexts = mode[:context] || []
+        #  next if contexts.empty?
+        total = context.sum {|kw| keyword_counts[kw.downcase] || 0}
+        scores[name] = total if total > 0
+      #  end
+      end
+      scores
+    end
+
+
+    def self.prefix_match?(profile, delim_prefix)
+      #prefixes = mode[:prefixes] || []
+      prefixes = profile[:prefixes] || []
+      return false if prefixes.empty?
+
+      delimiters =  "= : "
+      raw_prefix = delim_prefix.strip.split(/[#{Regexp.escape(delimiters)}]/).last&.strip&.downcase
+      prefixes.map(&:downcase).include?(raw_prefix)
+    end
+
+
+    def self.assign_confidence(scores_hash,  prefix_matched_mode=nil)
+      all_scores = scores_hash.values
+
+      return {} if all_scores.empty?
+
+      avg_score = all_scores.sum.to_f / all_scores.size
+     
+      scores_hash.transform_values do |score|
+        if score == 0
+          "regex-match"
+        else
+          mode_name = scores_hash.key(score)
+          is_prefix_mode = (prefix_matched_mode == mode_name)
+          deviation = (score - avg_score) / avg_score
+
+          case deviation
+          when 2.0..Float::INFINITY
+            "high"
+          when 0.5..2.0
+            is_prefix_mode ? "high" : "medium-high"
+          else
+            is_prefix_mode ? "medium-high" : "medium-low"
+          end
+        end 
+      end
+    end
+  end
+  #private_constant :Analyzer
+end