heitt 0.4.5 → 0.4.6

data/lib/heitt/formatter.rb

@@ -0,0 +1,125 @@
+require 'json'
+require_relative 'utils'
+
+module HEITT
+  module Formatter
+
+
+    def self.tree(groups, verbose: false, extended: false, show_regex_match: false)
+      result = ""
+
+      #Filter out groups with extended candidates as true
+      visible_groups = groups.select do |group|
+        has_non_extended = group[:candidates].any? {|c| !c[:extended] || extended}
+        has_non_regex = group[:candidates].any? {|c| c[:confidence] != "regex-match" || show_regex_match}
+        has_non_extended && has_non_regex
+      end
+      #Renumber after filtering
+      renumbered_groups= visible_groups.each_with_index.map { |group, index| group.merge(cluster_id: index + 1) }
+
+      root = {
+        text: "#{HEITT::Color.colorize("\n\n[", :bold, :blue)}#{HEITT::Color.colorize("CLUSTERED HASHES", :green)}#{HEITT::Color.colorize("]", :bold, :blue)}", 
+        children: renumbered_groups.map do |group|
+          {
+            text: HEITT::Color.colorize("HASH CLUSTER #{group[:cluster_id]}", :magenta, :bold),
+            children: group[:hashes].map{|h| {text: h, children: []}}
+          }
+        end
+      }
+
+      result += render_tree([root])
+
+      renumbered_groups.each do |group|
+        result += "#{HEITT::Color.colorize("\n\n[", :bold, :blue)}#{HEITT::Color.colorize("HASH CLUSTER #{group[:cluster_id]}", :white, :bold)}#{HEITT::Color.colorize("]\n", :bold, :blue)}"#, children: []}
+        candidate_nodes = (group[:candidates]).each_with_index.map do |candidate, idx|
+          next if candidate.nil?
+          next if candidate[:name].nil?
+          next if candidate[:extended] && !extended
+          next if candidate[:confidence] == "regex-match" && !show_regex_match
+          confidence = candidate[:confidence] ? " — CONFIDENCE: #{candidate[:confidence].upcase}" : ""
+
+          children = [
+            {text: "Hashcat Mode: #{candidate[:hashcat] || "--"}", children: []},
+            {text: "John Format: #{candidate[:john] || "--"}", children: []}
+          ]
+
+          if verbose
+            if candidate[:description] && !candidate[:description].empty?
+              children << {text: "Description: #{candidate[:description]}", children: []}
+            end
+
+            if candidate[:notes] && !candidate[:notes].empty?
+              children << {text: "Notes:", children: candidate[:notes].map {|note| {text: note, children: []}}}
+            end
+          end
+          {
+            text: "#{HEITT::Color.colorize("[", :bold, :blue)}#{HEITT::Color.colorize("CANDIDATE #{idx + 1}: ", :bold, :cyan)}#{HEITT::Color.colorize("#{candidate[:name]}#{confidence}", :bold, :cyan)}#{HEITT::Color.colorize("]", :bold, :blue)}",
+            children: children
+          }
+        end.compact
+        result += render_tree(candidate_nodes, "", false, false) unless candidate_nodes.nil? || candidate_nodes.empty?
+      end
+      result
+    end
+
+    def self.json(groups, extended: false, show_regex_match: false)
+      visible_groups = groups.select do |group|
+        has_non_extended = group[:candidates].any? {|c| c[:extended] || extended}
+        has_non_regex =  group[:candidates].any? {|c| c[:confidence] != "regex-match" || show_regex_match}
+        has_non_extended && has_non_regex
+      end
+      #Renumber after filtering
+      renumbered_groups = visible_groups.each_with_index.map { |group, index| group.merge(cluster_id: index+1)}
+
+      JSON.pretty_generate(
+        renumbered_groups.map do |group|
+          visible_candidates = group[:candidates].select do |c|
+            (!c[:extended] || extended)  && (c[:confidence] != "regex-match" || show_regex_match)
+          end
+          {
+           cluster_id: group[:cluster_id],
+           count: group[:count],
+            hashes: group[:hashes],
+            candidates: visible_candidates.map do |candidate|
+              {
+                name: candidate[:name],
+                hashcat: candidate[:hashcat],
+                john: candidate[:john],
+                confidence: candidate[:confidence],
+                description: candidate[:description]
+              }
+            end
+          }
+        end
+      )
+    end
+
+    private
+    def self.render_tree(items, prefix = "", parent_is_last=true, is_root=true)
+      result = ""
+
+      items.each_with_index do |node, i|
+        is_last_item = (i == items.length - 1)
+
+        line = if is_root
+          "#{node[:text]}\n"
+        else
+          "#{HEITT::Color.colorize(prefix, :blue)}#{HEITT::Color.colorize((is_last_item ? '└── ' : '├── '), :blue)}#{node[:text]}\n"
+        end
+
+        child_prefix = if is_root
+          ""
+        else
+          "#{HEITT::Color.colorize(prefix, :bold, :blue)}#{HEITT::Color.colorize((is_last_item ? "    " : "│   "), :bold, :blue)}"
+        end
+        result += line 
+        result += render_tree(node[:children], child_prefix, is_last_item, false) if node[:children].any?
+      
+        if is_last_item && !is_root and !node[:children].any?
+          result += "#{HEITT::Color.colorize(prefix, :bold, :blue)}  \n"
+        end
+      end
+      result
+    end
+  end
+end

data/lib/heitt/grouper.rb

@@ -0,0 +1,22 @@
+require_relative 'utils'
+module HEITT 
+  module Grouper
+
+    def self.group(results)
+      clusters = {}
+
+      clusters = results.group_by {|r| r[:candidates].first[:name]}
+      groups = clusters.each_with_index.map do |(name, group), index|
+        hashes = group.map {|r| r[:hash]}
+        {
+          cluster_id: index+1,
+          hashes: hashes,
+          candidates: group.first[:candidates],
+          count: hashes.size
+        }
+      end
+      HEITT::Logger.debug("Hashes grouped successfully") unless groups.empty? || groups.nil?
+      groups
+    end
+  end 
+end

data/lib/heitt/profiles.rb

@@ -0,0 +1,184 @@
+module HEITT
+  PROFILES = {
+    "CRC-16-CCITT" => {
+        description: "Cyclic Redundancy Check 16-bit Consultative Commitee for International Telegraph and Telephone",
+        notes: ["Used for error detection in communication and storage systems", "Data Integrity and verification", "Memory checks integrity", "Not cryptographic"],
+        context: ["checksum", "telecom", "bluetooth"], 
+        common_sources: ["V.41", "X.25", "HDLC", "Bluetooth"]
+    },
+    "CRC-16" => {
+        description: "Cyclic Redundancy Check 16-bit — 4 hexadecimal chars, basic checksum",
+        notes: ["Error detection in data transmission", "Data storage integrity checks", "Not cryptographic", "Low collision resistance"],
+        context: ["checksum", "networking"], 
+        prefixes: ["crc-16"],
+        common_sources: ["file verification", "network protocols", "embedded systems"]
+    },
+    "FCS-16" => {
+        description: "Frame Check Sequence 6-bit — 4 hexadecimal chars, data link layer",
+        notes: ["Not cryptographic"],
+        prefixes: ["fcs-16"],
+        context: ["checksum", "networking"], 
+        common_sources: ["Ethernet frames", "PPP"]
+    },
+    "Adler-32" => {
+        description: "Adler-32 checksum — 8 hex chars, zlib compression",
+        common_sources: ["zlib", "PNG files", "RSYNC"], 
+        context: ["checksum", "compression"]     
+    },
+    "CRC-32B" => {
+        description: "CRC-32 IEEE 802.3 variant — 8 hex chars, Ethernet standard" ,
+        notes: ["Not cryptographic"],
+        common_sources: ["Ethernet", "MPEG-2", "PKZIP"], 
+        context: ["checksum", "networking"]            
+    },
+    "FCS-32" => {
+        description: "Frame Check Sequence 32-bit — 8 hex chars, advanced networking",
+        common_sources: ["advanced networking protocols"], 
+        context: ["checksum", "networking"]   
+    },
+    "GHash-32-3" => {
+        description: "G-Hash 32-bit 3-round — 8 hex chars, experimental hash",
+        common_sources: ["research", "academic"], 
+        context: ["experimental"]
+    },
+    "GHash-32-5" => {
+        description: "G-Hash 32-bit 5-round — 8 hex chars, experimental hash",
+        common_sources: ["research", "academic"], 
+        context: ["experimental"]
+    },
+    "FNV-132" => {
+        description: "Fowler-Noll-Vo hash 32-bit — 8 hex chars, fast non-crypto hash",
+        common_sources: ["DNS", "database indexing", "hash tables"], 
+        context: ["checksum", "programming"]
+    },
+    "Fletcher-32" => {
+        description: "Fletcher's checksum 32-bit — 8 hex chars, error detection",
+        common_sources: ["OSTA UDF", "ISO/IEC 8473-1"], 
+        context: ["checksum", "storage"]
+    },
+    "Joaat" => {
+        description: "Jenkins one-at-a-time hash — 8 hex chars, simple string hash",
+        common_sources: ["Perl", "Apache", "various applications"], 
+        context: ["programming", "hashing"]
+    },
+    "ELF-32" => {
+        description: "ELF-32 hash for object files — 8 hex chars, Unix/Linux object files",
+        context: ["executable", "system"],
+        mime_types: ["application/octet-stream"]
+    },
+    "XOR-32" => {
+        description: "Simple XOR-based 32-bit hash — 8 hex chars, basic XOR operation",
+        common_sources: ["simple applications", "embedded systems"], 
+        context: ["basic", "embedded"]
+    },
+    "CRC-24" => {
+        description: "Cyclic Redundancy Check 24-bits — 6 hexadecimal chars, OpenPGP standard",
+        notes: ["Not cryptographic"],
+        context: ["checksum"], 
+        common_sources: ["OpenPGP", "RFID", "some file formats"]        
+    },
+    "CRC-32" => {
+        description: "Cyclic Redundancy Check 32-bit — 8 hex chars, most common checksum",
+        notes: ["Not cryptographic"]
+    },
+    "DES(Unix)" => {
+        description: "DES-based Unix crypt — 13 chars, traditional Unix passwords",
+        notes: ["Only 8 char passwords", "weak salt"],
+        common_sources: ["/etc/passwd", "old Unix systems"], 
+        context: ["unix", "legacy"]   
+    },
+    "DEScrypt" => {
+        description: "DES crypt implementation — 13 chars",
+        notes: ["Traditional Unix password hashing"],
+        common_sources: ["old Unix/Linux"], 
+        context: ["unix", "legacy"]
+    },
+    "MySQL323" => {
+        description: "MySQL 3.23 password hash — 16 chars typical, but can be padded to 32 (hexadecimals)",
+        notes: ["Used in old MySQL databases", "Can be broken in seconds", "Susceptible to rainbow tables", "Limited to 8 character passwords", "Deprecated since MySQL 4.1"]
+    },
+    "DES(Oracle)" => {
+        description: "Oracle DES-based hash — 16 hex chars, Oracle specific"
+    },
+    "Half MD5" => {
+        description: "First half of MD5 hash — 16 hex chars, MD5 truncated",
+        notes: ["Weaker than full MD5"]
+    },
+    "FNV-164" => {
+        description: "Fowler-Noll-Vo hash 64-bit —  16 hex chars, 64-bit version",
+        notes: ["Not cryptographic"]
+    },
+    "CRC-64" => {
+        description: "Cyclic Redundancy Check 64-bit — 16 hex chars, ISO 3309",
+        notes: ["Not cryptographic"]
+    },
+    "CRC-96(ZIP)" => {
+        description: "CRC-96 used in some ZIP variants — 24 hex chars, extended CRC",
+        notes: ["Not cryptographic", "For some archive formats"]
+    },
+    "Crypt16" => {
+        description: "Extended crypt16 implementation", 
+        characteristics: "24 chars, extended DES crypt",
+        notes: ["Rarely used", "Used by some Unix variants"]
+    },
+    "BigCrypt" => {
+         description: "Extended DES crypt — 13+ chars, extended length",
+        notes: ["Rarely used", "Used in some Unix variants"], 
+        common_sources: ["some Unix variants"], 
+        context: ["unix", "extended"]
+    },
+    "MD5" => {
+        description: "MD5 cryptographic hash function",
+        characteristics: "32 chars, hexadecimal, unsalted",
+        notes: ["Used as checksum to verify data or file integrity", "MD5 is cryptographically broken as it is vulnerable to collision attacks"],
+        context: ["web", "checksum", "legacy", "password", "hash", "md5"],    
+        prefixes: ["md5", "hash", "checksum", "password"],
+        file_types: ["shadow", "htpasswd", "logs"],
+        mime_types: ["text/plain", "text/x-passwd"],
+        common_sources: ["web applications", "file integrity checks", "checksums", "legacy systems"]
+    },
+    "MD4" => {
+        characteristics: "32 chars, legacy Microsoft systems",
+        prefixes: ["hash"],
+        context: ["hash"],
+        common_sources: ["Old Windows systems", "legacy applications"]
+    },
+    "LM" => {
+        description: "Windows LAN Manager hash", 
+        characteristics: "16 hex chars, all uppercase, split password",
+        notes: ["Mainly found in Windows SAM files(legacy Windows)", "Very weak", "no lowercase", "split passwords"],
+        common_sources: ["Windows SAM", "legacy Windows systems"], 
+        context: ["windows", "SAM"]
+    },
+    "NTLM" => {
+        description: "Windows NTLM authentication hash",
+        characteristics: "32 chars, Windows authentication, based on MD4",
+        notes: ["Hashcat Mode: 5600 (NetNTLMv2) - if network captured", "Hashcat Mode: 5500 (NetNTLMv1/NetNTLMv1+ESS) - legacy versions", "John Format: netntlm (for network hashes)", "John Format: netntlmv2 (v2 hashes)"],
+        context: ["windows", "SAM", "LSASS", "nt", "ntlm"],
+        prefixes: ["nt"],
+        file_types: ["ntds", "logs"],
+        mime_types: ["text/plain", "application/octet-stream"],
+        common_sources: ["Windows SAM", "Active Directory", "LSASS memory"]
+    },
+    "SHA-1" => {
+        description: "SHA-1 cryptographic hash function", 
+        characteristics: "40 chars, hexadecimal, unsalted",
+        notes: ["Used for file verification", "found in git commits and legacy certificates"],
+        prefixes: ["sha1", "hash"],
+        context: ["sha1", "hash"]
+    },
+    "RIPEMD-160" => {
+        characteristics: "40 chars, Bitcoin addresses, digital signatures",
+        notes: ["Rarely used for passwords"]
+    },
+    "Android PIN" => {
+        description: "Android PIN/Password hash", 
+        characteristics: "40 chars hash + 16 chars salt, SHA1 + MD5",
+        notes: ["found in android gesture.key files"]
+    },
+    "SHA-512 Crypt" => {
+        characteristics: "$6$ prefix, includes salt, 96-106 chars", 
+        notes: ["Industry standard for modern Linux systems"]
+    },
+  }
+end

data/lib/heitt/scanner.rb

@@ -0,0 +1,65 @@
+require 'strscan'
+require 'set'
+require_relative 'analyzer'
+require_relative 'database'
+require_relative 'profiles'
+
+module HEITT
+  module Scanner
+
+    def self.scan(input, database: HEITT::DATABASE, profiles: HEITT::PROFILES, min_entropy: 3.5)
+      text = File.exist?(File.expand_path(input)) ? File.read(File.expand_path(input)) : input
+      context_scores = HEITT::Analyzer.analyze(text, profiles: profiles)#database: database)
+      found = {}
+      #seen = {}
+     
+
+      database.each do |entry|
+        regex = get_regex(entry)
+        modes = get_modes(entry)
+        next unless regex && modes && !modes.empty?
+        pattern = regex.is_a?(Regexp) ? regex : Regexp.new(regex)
+        scanner = StringScanner.new(text)      
+        
+        while scanner.scan_until(pattern)
+          matched = scanner.matched 
+          next unless matched.length < 8 || HEITT::Analyzer.high_entropy?(matched, min_entropy)
+          offset = scanner.pos - matched.length
+          HEITT::Logger.debug("Extracting prefix..")
+          delim_prefix = HEITT::Analyzer.extract_prefix(text, offset)
+          HEITT::Logger.debug("Extracted prefix: #{delim_prefix.length <= 1 ? "NULL" : delim_prefix}")
+
+          candidates = HEITT::Analyzer.score_candidates(modes, delim_prefix, context_scores)
+          #score = candidates.first[:score]
+
+          found[matched] ||= {hash: matched, candidates: []}
+          found[matched][:candidates].concat(candidates)
+          end
+        end
+
+          found.each_value do |result|
+           result[:candidates] = result[:candidates]
+            .group_by {|c| c[:name]}
+            .map {|name, dupes| dupes.max_by {|c| c[:score]}}
+            .sort_by {|c| -c[:score]}
+
+            # Re-assign confidence based on final merged scores
+            scores_hash = result[:candidates].map {|c| [c[:name], c[:score]]}.to_h
+            confidences = Analyzer.assign_confidence(scores_hash)
+            result[:candidates] = result[:candidates].map {|c| c.merge(confidence: confidences[c[:name]])}
+      end
+      found.values
+    end
+
+    private
+    
+    def self.get_regex(entry)
+      entry[:extract_regex] || entry[:regex] || entry[:pattern] || entry[:regexp]
+    end
+
+    def self.get_modes(entry)
+      entry[:modes] || entry[:algorithms] || entry[:hashes] || 
+      entry[:candidates] || entry[:types] || entry[:hashtypes]
+    end
+  end
+end

data/lib/heitt/utils.rb

@@ -0,0 +1,57 @@
+require 'logger'
+require 'colorize'
+
+module HEITT
+  module Logger
+    @debug = false
+    #LEVELS = { debug: 0, verbose: 1, info: 2, warn: 3, error: 4 }
+    #@level = :warn 
+
+    #def self.set_level(lvl)
+    #  @level = lvl
+    #end
+    def self.enable_debug
+      @debug = true
+    end
+
+    def self.disable_debug
+      @debug = false
+    end
+
+    def self.debug_enabled?
+      @debug
+    end
+
+    def self.debug(msg)
+      log("[DEBUG] #{msg}", :cyan)
+    end
+
+    def self.warn(msg)
+      log("[WARN] #{msg}", :yellow)
+    end
+
+    def self.error(msg)
+      log("[ERROR] #{msg}", :red)
+    end
+
+    private
+    def self.log(msg, color)
+      return unless @debug
+      $stderr.puts HEITT::Color.colorize(msg, color)
+    end
+  end
+
+
+  module Color
+    def self.colorize(text, color, *styles)
+      return text unless STDOUT.isatty #&& !(defined?(Flags) && Flags.no_color)
+
+      colored = text.colorize(color)
+      styles.each do |style|
+        colored = colored.send(style)
+      end
+      colored
+    end
+  end
+  #private_constant :Color
+end

data/lib/heitt/version.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
 module HEITT
-  VERSION = "0.4.5"
+  VERSION = "0.4.6"
   GITHUB = "https://github.com/jobotow/heitt"
 end