heitt 0.4.5 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: aaae42753b6fd3c04375200d118517b8b3f45eb42d0bf6b4fddc9b08c64e5af7
-  data.tar.gz: ed0b83ef1116ebc291b000985784107a709156df68be16443598d2a41ae59853
+  metadata.gz: ae7835969de375994bf7a0f9bf03c453f1ccd77d9086b4712a13e755084cb01c
+  data.tar.gz: 5754d899e34e5a9e8162c451fcef359939165221a0a758348e9ec608e9c0cc5a
 SHA512:
-  metadata.gz: c4f7ea072d396d4c5ba96f9f65402ab006c1a94856b77cd359417d7ddc26bde65bc81928369f64ad36562aa0d7ce4e7b0efbe4d04edb6d04e467c9df4ce5ab6d
-  data.tar.gz: 3a1c7494156de04f10c2a3f132f85337fdd597197883b81cc4d1df934fd3c9d70dcaf89378729837a7e0f086d6cb2d2c75518675e0af56b60e4ee7c3e2f1f804
+  metadata.gz: 1f2174d59e403b16385946ca733cdc428fb75f09c6c4275f42d6ecb6870725f8a4224d8389934d4d5a0106e6a01f65d9226eb1de30fc6997d98e2b4c81b09ade
+  data.tar.gz: 228b63e61c87e07b4f8f5c68afc8121fe95c6e9bffd644408cda410e05d7e6314684d730e14fa640793c14b8d21dc7664ba22ed84c27b5399fdf99141e249c2d

data/bin/heitt

@@ -2,19 +2,19 @@
 require 'optparse'
 require 'io/console'
 require 'heitt'
+#require_relative '../lib/heitt'
 
 
 
 module HEITT
   class CLI
-    attr_accessor :inputs, :extended, :verbose, :output, :json, :database, :show_regex_match, :min_entropy
-
+    attr_accessor :inputs, :extended, :verbose, :tree, :database, :show_regex_match, :min_entropy, :peel_limit
     def initialize
       @inputs = []
       @extended = false
       @verbose = false
-      @output = ""
-      @json = false
+      @tree = false
+      @peel_limit = 0
       @database = ""
       @show_regex_match = false
       @min_entropy = 3.5
@@ -45,8 +45,12 @@ module HEITT
         opts.separator ""
         opts.separator "OUTPUT OPTIONS:"
         opts.on("-V", "--verbose", "Show description and notes for each candidate") {@verbose = true}
-        opts.on("-j", "--json","Output in json format") {@json = true}
-        opts.on("-o", "--output FILEPATH", String, "File to write output to") {|v| @output = v}
+        opts.on("-t", "--tree", "Force tree format when output is redirected"){@tree = true}
+        opts.on("--debug", "Output in debug mode") {HEITT::Logger.enable_debug}
+
+        opts.separator ""
+        opts.separator "DECODE OPTIONS:"
+        opts.on("-L", "--peel-limit INTEGER", Integer, "Maximum decode layers to peel(supports base64 and hex)") {|v| @peel_limit = v}
 
         opts.separator ""
         opts.separator "FILTERING OPTIONS:"
@@ -62,7 +66,7 @@ module HEITT
         opts.separator "EXAMPLES:"
         opts.separator "    heitt 634d398e96eb1550956b8128cfeb0747 -r"
         opts.separator "    heitt auth.log"
-        opts.separator "    heitt auth.log --json --output result.json"
+        opts.separator "    heitt auth.log | cat"
         opts.separator "    heitt auth.log --extended --regex-match"
         opts.separator "    heitt auth.log --min-entropy 4.0"
         opts.separator "    cat auth.log | heitt "
@@ -70,6 +74,7 @@ module HEITT
         opts.separator ""
         opts.separator "NOTES:"
         opts.separator "    JSON format is default when output is redirected or piped."
+        opts.separator "    Custom database if merged with default database"
         opts.separator "    Regex-match candidates are hidden by default, use '-r' to show."
         opts.separator "    Running without input starts interactive mode."
         opts.separator "#{header("=========================================================================")}"
@@ -78,30 +83,13 @@ module HEITT
       end.parse!
 
       @inputs = if ARGV.empty?
+
         if $stdin.tty?
-          # Interactive mode
-          puts "#{header("=== HEITT INTERACTIVE MODE ====\n")}"
-          puts "Enter a hash or file"
-          loop do
-            print "heitt> "
-            input = $stdin.gets&.strip
-            break if input.nil? || input.empty?
-            results = HEITT::Scanner.scan(input, min_entropy: @min_entropy)
-            groups = HEITT::Grouper.group(results)
-            format = @json ? :json : :tree
-            output = case format
-            when :json
-              HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
-            else
-              HEITT::Formatter.tree(groups, extended: @extended, show_regex_match: @show_regex_match)
-            end
-            puts output
-            puts ""
-          end
+          # instead of hanging, make it interactive
+          interactive_loop
         else
           [$stdin.read]
         end
-
         
       else
         ARGV.dup
@@ -109,69 +97,119 @@ module HEITT
       self
     end
 
+    def interactive_loop
+      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
+      # Interactive mode
+      puts header("=== HEITT INTERACTIVE MODE ====")
+      #print "[debug:#{HEITT::Logger.debug_enabled? ? "on": "off"}  ||  "
+      print "extended: #{@extended ? "on": "off"}  ||  "
+      print "peel-limit: #{@peel_limit}  ||  "
+      print "verbose: #{@verbose ? "on": "off"}  ||  "
+      print "regex-match: #{@show_regex_match ? "on": "off"}  ||  "
+      print "min-entropy: #{@min_entropy}  ||  "
+      print "database: #{@database.empty? ? "default": database}  ||  "
+      print "output-format: #{@json ? "json": "tree"}]\n"
+      puts ""
+      puts "Enter a hash or filepath. Press Ctrl+C or Enter to exit"
+      #puts "Extended: true" unless 
+      # puts used flags here
+          
+      loop do
+        print "heitt> "
+        input = $stdin.gets&.strip
+        break if input.nil? || input.empty?
+        results = HEITT::Scanner.scan(input, min_entropy: @min_entropy)
+        groups = HEITT::Grouper.group(results)
+
+        format = :tree
+
+        output = case format
+        when :tree
+          HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
+        else
+          HEITT::Formatter.tree(groups, verbose: @verbose, extended: @extended, show_regex_match: @show_regex_match)
+        end
+        puts output
+        puts ""
+      end
+    end
+
+
     def run 
-      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)
+      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
       
-      format = @json || !$stdout.tty? ? :json : :tree
+      format = if @tree
+        :tree
+      elsif !$stdout.tty?
+        :json
+      else 
+        :tree
+      end
+
       output = @inputs.map do |input|
-        results = HEITT::Scanner.scan(input, database: database, min_entropy: @min_entropy)
+        text = read_input(input)
+        text = HEITT::Decoder.peel(text, peel_limit: @peel_limit)[:final] if @peel_limit > 0
+        results = HEITT::Scanner.scan(text, database: database, min_entropy: @min_entropy)
         groups = HEITT::Grouper.group(results)
 
         case format
         when :json
           HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
         else
-          HEITT::Formatter.tree(groups, extended: @extended, show_regex_match: @show_regex_match)
+          HEITT::Formatter.tree(groups, verbose: @verbose, extended: @extended, show_regex_match: @show_regex_match)
         end
       end.join("\n")
 
-      unless @output.empty?
-        File.write(@output, output)
-      else
-        puts output
-      end
+      #unless @output.empty?
+      #  File.write(@output, output)
+      #else
+      puts output
+      #end
     end
     
     private
+    def read_input(input)
+      File.exist?(File.expand_path(input)) ? File.read(File.expand_path(input)) : input
+    end
+
     def header(text)
       terminal_width = IO.console.winsize[1] rescue 70
       text.center(terminal_width)
     end
 
 
-    def load_custom_database(filepath)
+
+    def load_custom_database(file_path)
+      filepath = File.expand_path(file_path)
+      HEITT::Logger.debug("filepath: #{filepath}")
       unless File.exist?(filepath)
-        puts HEITT::Color.colorize("[ERROR] Database file #{filepath} does not exists", :bold, :red)
+        HEITT::Logger.error("file #{filepath} does not exist")
         exit(1)
       end
       begin
         custom_database = JSON.parse(File.read(filepath), symbolize_names: true)
         HEITT::DATABASE +  custom_database
       rescue JSON::ParserError => e 
-        puts "#{HEITT::Color.colorize("[ERROR] Invalid JSON in database file: ", :bold, :red)}#{e.message}"
+        HEITT::Logger.error("Invalid JSON in #{label} file: #{e.message}")
         exit(1)
       end
     end
-
-  #  def decode(text, decoders)
-      #Check if result is printable ascii after decoding
-   #   decoders.each do |decoder|
-    #    case decoder
-    #    when "b64"
-    #      text = text.gsub(/[A-Za-z0-9+\/]{4,}={0,2}/) do |match|
-    #        current = match
-    #        loop do
-    #          #until word is non printable ascii continue decoding
-     #         decoded = Base64.decode64(word) #rescue break
-     #         current = decoded.strip.match?(/\A[[:print:]]+\z/) ? decoded.strip : break
-              #puts "IS PRINTABLE?: #{decoded.match?(/\A[[:print:]]+\z/)}"
-              #puts "RESULT: #{result}"
-     #       end
-      #      current
-       #   end
-       # when "hex"
-        #  text = tesxt.gsub(/)
   end
 end
 
 cli = HEITT::CLI.new.parse!.run #if __FILE__ == $0
+
+## Some flags will be removed.
+# output flag will be replaced by tree flag. it existed because
+# tree couldnt show when output was redirected.
+# json flag will be removed, although it was created  so its format will be seen
+# it now looks redundant to the automatic version.
+# -r flag wont be used for everything. it was made
+# to filter out low confidence types from  high confidence ones and show when asked for.
+# in case of all being "regex-matched", -r shouldnt be necessary.
+# this will make it convenient to check single hashes without always needing -r flag
+#auto decoding hasn't being dropped, a library "codeberg.org/jobotow/lookslike" is in the making to help with encoded string identification.
+# it will decode and check it it's a binary blob. if first decode shows binary blob, then it certainly isn't a hash hex or proper b64.
+# else it should continue decoding till ut also meets binary blob before returning back to previous text before last decode"
+# from the context that a text that's encoded once cant be decoded twice because the binary blob will show that the second text undergoing decoding was the actual answer.
+# binary blob will be the eyes as it's uncertain the number of times a text has being encoded.

data/lib/heitt/analyzer.rb

@@ -0,0 +1,153 @@
+require_relative 'profiles'
+
+module HEITT
+  module Analyzer
+    def self.analyze(text, profiles: HEITT::PROFILES)#database: HEITT::DATABASE)
+      keyword_counts = keyword_counts(text.downcase, profiles: profiles)
+      algorithm_scores(keyword_counts, profiles: profiles)
+    end
+
+    def self.extract_prefix(text, offset)
+      line_start = text.rindex("\n", offset) || 0
+      text[line_start...offset]
+    end
+
+    def self.high_entropy?(text, min_ent)
+      entropy(text) >= min_ent
+    end
+      
+
+    def self.score_candidates(modes, delim_prefix, context_scores, profiles: HEITT::PROFILES)
+      prefix_matched_mode = nil
+      #context based scoring
+      matches = modes.map do |mode|
+        profile = profiles[mode[:name]] || {}
+        score = context_scores[mode[:name]] || 0
+        #score = score_data || 0
+
+        if prefix_match?(profile, delim_prefix)
+          #boost score as confidence is high if prefix matched
+          prefix_matched_mode = mode[:name]
+          score += 20
+        end
+        {
+          name: mode[:name],
+          hashcat: mode[:hashcat],
+          john: mode[:john],
+          extended: mode[:extended],
+          description: profile[:description],
+          notes: profile[:notes],
+          score: score
+        }
+      end 
+      return [] if matches.empty?
+
+      #calculate confidence
+      scores_hash = matches.map {|m| [m[:name], m[:score]]}.to_h 
+      
+      confidences = assign_confidence(scores_hash, prefix_matched_mode)
+      scored_candidates = matches.map{|m| m.merge(confidence: confidences[m[:name]])}.sort_by {|m| -m[:score]}
+      scored_candidates
+    end
+
+
+    private
+    #def self.get_modes(entry)
+    #  entry[:modes] || entry[:algorithms] || entry[:hashes] || 
+    #  entry[:candidates] || entry[:types] || entry[:hashtypes]
+    #end
+
+    #this  code is an inspiration of "https://github.com/chrisjchandler/entropy/blob/main/entropy.go"
+    def self.entropy(text)
+      frequency = Hash.new(0)
+      text.each_char { |ch| frequency[ch] += 1 }
+
+      #calculate the total number of characters
+      total = text.length.to_f
+      #caluclate entropy
+      entropy = 0.0
+      frequency.each_value do |count|
+        probability = count.to_f / total
+        entropy += probability * Math.log2(probability)
+      end
+      #negate the sum as entropy is positive
+      -entropy
+    end
+    
+    def self.keyword_counts(content_lower, profiles: HEITT::PROFILES) #HEITT::DATABASE)
+      keywords = profiles.values.flat_map { |p| p[:context] || [] }.uniq.map(&:downcase)
+      #database.flat_map do |entry|
+        #modes = get_modes(entry)
+        #next [] unless modes
+        #modes.flat_map {|mode| HEITT::PROFILES[mode[:name]][:context] || []}
+      #end#.uniq.map(&:downcase)
+
+      counts = {}
+      keywords.each do |kw|
+        count = content_lower.scan(/\b#{Regexp.escape(kw)}\b/).size
+        counts[kw] = count if count > 0
+      end
+      counts
+    end
+
+
+    def self.algorithm_scores(keyword_counts, profiles: HEITT::PROFILES)#database: HEITT::DATABASE)
+      scores = {}
+      return scores if keyword_counts.nil?
+
+      profiles.each do |name, profile| #database.each do |entry|
+        context = profile[:context] || []
+        next if context.empty?
+        #modes = get_modes(entry)
+        #next unless modes
+        #modes.each do |mode|
+        #  contexts = mode[:context] || []
+        #  next if contexts.empty?
+        total = context.sum {|kw| keyword_counts[kw.downcase] || 0}
+        scores[name] = total if total > 0
+      #  end
+      end
+      scores
+    end
+
+
+    def self.prefix_match?(profile, delim_prefix)
+      #prefixes = mode[:prefixes] || []
+      prefixes = profile[:prefixes] || []
+      return false if prefixes.empty?
+
+      delimiters =  "= : "
+      raw_prefix = delim_prefix.strip.split(/[#{Regexp.escape(delimiters)}]/).last&.strip&.downcase
+      prefixes.map(&:downcase).include?(raw_prefix)
+    end
+
+
+    def self.assign_confidence(scores_hash,  prefix_matched_mode=nil)
+      all_scores = scores_hash.values
+
+      return {} if all_scores.empty?
+
+      avg_score = all_scores.sum.to_f / all_scores.size
+     
+      scores_hash.transform_values do |score|
+        if score == 0
+          "regex-match"
+        else
+          mode_name = scores_hash.key(score)
+          is_prefix_mode = (prefix_matched_mode == mode_name)
+          deviation = (score - avg_score) / avg_score
+
+          case deviation
+          when 2.0..Float::INFINITY
+            "high"
+          when 0.5..2.0
+            is_prefix_mode ? "high" : "medium-high"
+          else
+            is_prefix_mode ? "medium-high" : "medium-low"
+          end
+        end 
+      end
+    end
+  end
+  #private_constant :Analyzer
+end