heitt 0.4.6 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2682eb074a7763036e54775706be2ae16221acd2d4cd7d416b387723e1b51aa2
-  data.tar.gz: 4739a64851b2e071217eed0b12e3e71327670a4abcdebd218fdb3020df4aa47a
+  metadata.gz: ae7835969de375994bf7a0f9bf03c453f1ccd77d9086b4712a13e755084cb01c
+  data.tar.gz: 5754d899e34e5a9e8162c451fcef359939165221a0a758348e9ec608e9c0cc5a
 SHA512:
-  metadata.gz: d6084258bca3bc35f40aecf9402dffbace9a419561f886e02a73939e821a7369798d4878b3e6483b4293687082ff169f7206c5046d94fc64ad38085718c758f5
-  data.tar.gz: 362c46e00ba6dde92e1763e39b2f965e7782fb46a2073ab25974bcbb9d9213bfa03719b30b6747a145e9d8fdf8bc200e26a9c8df88c8c1022b4657147ed13f6e
+  metadata.gz: 1f2174d59e403b16385946ca733cdc428fb75f09c6c4275f42d6ecb6870725f8a4224d8389934d4d5a0106e6a01f65d9226eb1de30fc6997d98e2b4c81b09ade
+  data.tar.gz: 228b63e61c87e07b4f8f5c68afc8121fe95c6e9bffd644408cda410e05d7e6314684d730e14fa640793c14b8d21dc7664ba22ed84c27b5399fdf99141e249c2d

data/bin/heitt

@@ -8,14 +8,13 @@ require 'heitt'
 
 module HEITT
   class CLI
-    attr_accessor :inputs, :extended, :verbose, :output, :json, :database, :show_regex_match, :min_entropy
-
+    attr_accessor :inputs, :extended, :verbose, :tree, :database, :show_regex_match, :min_entropy, :peel_limit
     def initialize
       @inputs = []
       @extended = false
       @verbose = false
-      @output = ""
-      @json = false
+      @tree = false
+      @peel_limit = 0
       @database = ""
       @show_regex_match = false
       @min_entropy = 3.5
@@ -46,10 +45,13 @@ module HEITT
         opts.separator ""
         opts.separator "OUTPUT OPTIONS:"
         opts.on("-V", "--verbose", "Show description and notes for each candidate") {@verbose = true}
-        opts.on("-j", "--json","Output in json format") {@json = true}
-        opts.on("-o", "--output FILEPATH", String, "File to write output to") {|v| @output = v}
+        opts.on("-t", "--tree", "Force tree format when output is redirected"){@tree = true}
         opts.on("--debug", "Output in debug mode") {HEITT::Logger.enable_debug}
 
+        opts.separator ""
+        opts.separator "DECODE OPTIONS:"
+        opts.on("-L", "--peel-limit INTEGER", Integer, "Maximum decode layers to peel(supports base64 and hex)") {|v| @peel_limit = v}
+
         opts.separator ""
         opts.separator "FILTERING OPTIONS:"
         opts.on("-e", "--extended", "Show extended candidates") {@extended = true}
@@ -64,7 +66,7 @@ module HEITT
         opts.separator "EXAMPLES:"
         opts.separator "    heitt 634d398e96eb1550956b8128cfeb0747 -r"
         opts.separator "    heitt auth.log"
-        opts.separator "    heitt auth.log --json --output result.json"
+        opts.separator "    heitt auth.log | cat"
         opts.separator "    heitt auth.log --extended --regex-match"
         opts.separator "    heitt auth.log --min-entropy 4.0"
         opts.separator "    cat auth.log | heitt "
@@ -81,42 +83,13 @@ module HEITT
       end.parse!
 
       @inputs = if ARGV.empty?
+
         if $stdin.tty?
-          database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
-          # Interactive mode
-          puts header("=== HEITT INTERACTIVE MODE ====")
-          print "[debug:#{HEITT::Logger.debug_enabled? ? "on": "off"}  ||  "
-          print "extended: #{@extended ? "on": "off"}  ||  "
-          print "verbose: #{@verbose ? "on": "off"}  ||  "
-          print "regex-match: #{@show_regex_match ? "on": "off"}  ||  "
-          print "min-entropy: #{@min_entropy}  ||  "
-          print "database: #{@database.empty? ? "default": database}  ||  "
-          print "output-format: #{@json ? "json": "tree"}]\n"
-          puts ""
-          puts "Enter a hash or filepath. Press Ctrl+C or Enter to exit"
-          #puts "Extended: true" unless 
-          # puts used flags here
-          
-          loop do
-            print "heitt> "
-            input = $stdin.gets&.strip
-            break if input.nil? || input.empty?
-            results = HEITT::Scanner.scan(input, min_entropy: @min_entropy)
-            groups = HEITT::Grouper.group(results)
-            format = @json ? :json : :tree
-            output = case format
-            when :json
-              HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
-            else
-              HEITT::Formatter.tree(groups, extended: @extended, show_regex_match: @show_regex_match)
-            end
-            puts output
-            puts ""
-          end
+          # instead of hanging, make it interactive
+          interactive_loop
         else
           [$stdin.read]
         end
-
         
       else
         ARGV.dup
@@ -124,35 +97,81 @@ module HEITT
       self
     end
 
+    def interactive_loop
+      database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
+      # Interactive mode
+      puts header("=== HEITT INTERACTIVE MODE ====")
+      #print "[debug:#{HEITT::Logger.debug_enabled? ? "on": "off"}  ||  "
+      print "extended: #{@extended ? "on": "off"}  ||  "
+      print "peel-limit: #{@peel_limit}  ||  "
+      print "verbose: #{@verbose ? "on": "off"}  ||  "
+      print "regex-match: #{@show_regex_match ? "on": "off"}  ||  "
+      print "min-entropy: #{@min_entropy}  ||  "
+      print "database: #{@database.empty? ? "default": database}  ||  "
+      print "output-format: #{@json ? "json": "tree"}]\n"
+      puts ""
+      puts "Enter a hash or filepath. Press Ctrl+C or Enter to exit"
+      #puts "Extended: true" unless 
+      # puts used flags here
+          
+      loop do
+        print "heitt> "
+        input = $stdin.gets&.strip
+        break if input.nil? || input.empty?
+        results = HEITT::Scanner.scan(input, min_entropy: @min_entropy)
+        groups = HEITT::Grouper.group(results)
+
+        format = :tree
+
+        output = case format
+        when :tree
+          HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
+        else
+          HEITT::Formatter.tree(groups, verbose: @verbose, extended: @extended, show_regex_match: @show_regex_match)
+        end
+        puts output
+        puts ""
+      end
+    end
+
+
     def run 
-      HEITT::Logger.debug("Loading database...")
       database = @database.empty? ? HEITT::DATABASE : load_custom_database(@database)  
-      HEITT::Logger.debug("Database (#{@database.empty? ? "default" : @database}) loaded")
       
-      format = @json || !$stdout.tty? ? :json : :tree
+      format = if @tree
+        :tree
+      elsif !$stdout.tty?
+        :json
+      else 
+        :tree
+      end
+
       output = @inputs.map do |input|
-        HEITT::Logger.debug("Scanning #{input}...")
-        results = HEITT::Scanner.scan(input, database: database, min_entropy: @min_entropy)
-        HEITT::Logger.debug("Scanning completed (#{input})")
-        HEITT::Logger.debug("Grouping Results ...")
+        text = read_input(input)
+        text = HEITT::Decoder.peel(text, peel_limit: @peel_limit)[:final] if @peel_limit > 0
+        results = HEITT::Scanner.scan(text, database: database, min_entropy: @min_entropy)
         groups = HEITT::Grouper.group(results)
 
         case format
         when :json
           HEITT::Formatter.json(groups, extended: @extended, show_regex_match: @show_regex_match)
         else
-          HEITT::Formatter.tree(groups, extended: @extended, show_regex_match: @show_regex_match)
+          HEITT::Formatter.tree(groups, verbose: @verbose, extended: @extended, show_regex_match: @show_regex_match)
         end
       end.join("\n")
 
-      unless @output.empty?
-        File.write(@output, output)
-      else
-        puts output
-      end
+      #unless @output.empty?
+      #  File.write(@output, output)
+      #else
+      puts output
+      #end
     end
     
     private
+    def read_input(input)
+      File.exist?(File.expand_path(input)) ? File.read(File.expand_path(input)) : input
+    end
+
     def header(text)
       terminal_width = IO.console.winsize[1] rescue 70
       text.center(terminal_width)
@@ -162,9 +181,9 @@ module HEITT
 
     def load_custom_database(file_path)
       filepath = File.expand_path(file_path)
-      HEITT::Logger.debug("#{label} filepath: #{filepath}")
+      HEITT::Logger.debug("filepath: #{filepath}")
       unless File.exist?(filepath)
-        HEITT::Logger.error("#{label} file #{filepath} does not exist")
+        HEITT::Logger.error("file #{filepath} does not exist")
         exit(1)
       end
       begin
@@ -175,26 +194,22 @@ module HEITT
         exit(1)
       end
     end
-
-  #  def decode(text, decoders)
-      #Check if result is printable ascii after decoding
-   #   decoders.each do |decoder|
-    #    case decoder
-    #    when "b64"
-    #      text = text.gsub(/[A-Za-z0-9+\/]{4,}={0,2}/) do |match|
-    #        current = match
-    #        loop do
-    #          #until word is non printable ascii continue decoding
-     #         decoded = Base64.decode64(word) #rescue break
-     #         current = decoded.strip.match?(/\A[[:print:]]+\z/) ? decoded.strip : break
-              #puts "IS PRINTABLE?: #{decoded.match?(/\A[[:print:]]+\z/)}"
-              #puts "RESULT: #{result}"
-     #       end
-      #      current
-       #   end
-       # when "hex"
-        #  text = tesxt.gsub(/)
   end
 end
 
 cli = HEITT::CLI.new.parse!.run #if __FILE__ == $0
+
+## Some flags will be removed.
+# output flag will be replaced by tree flag. it existed because
+# tree couldnt show when output was redirected.
+# json flag will be removed, although it was created  so its format will be seen
+# it now looks redundant to the automatic version.
+# -r flag wont be used for everything. it was made
+# to filter out low confidence types from  high confidence ones and show when asked for.
+# in case of all being "regex-matched", -r shouldnt be necessary.
+# this will make it convenient to check single hashes without always needing -r flag
+#auto decoding hasn't being dropped, a library "codeberg.org/jobotow/lookslike" is in the making to help with encoded string identification.
+# it will decode and check it it's a binary blob. if first decode shows binary blob, then it certainly isn't a hash hex or proper b64.
+# else it should continue decoding till ut also meets binary blob before returning back to previous text before last decode"
+# from the context that a text that's encoded once cant be decoded twice because the binary blob will show that the second text undergoing decoding was the actual answer.
+# binary blob will be the eyes as it's uncertain the number of times a text has being encoded.

data/lib/heitt/analyzer.rb

@@ -1,12 +1,9 @@
-require_relative 'utils'
 require_relative 'profiles'
 
 module HEITT
   module Analyzer
     def self.analyze(text, profiles: HEITT::PROFILES)#database: HEITT::DATABASE)
-      HEITT::Logger.debug("Counting keywords...")
       keyword_counts = keyword_counts(text.downcase, profiles: profiles)
-      HEITT::Logger.debug("Counted keywords: #{keyword_counts}")
       algorithm_scores(keyword_counts, profiles: profiles)
     end
 
@@ -39,6 +36,7 @@ module HEITT
           john: mode[:john],
           extended: mode[:extended],
           description: profile[:description],
+          notes: profile[:notes],
           score: score
         }
       end 
@@ -49,7 +47,6 @@ module HEITT
       
       confidences = assign_confidence(scores_hash, prefix_matched_mode)
       scored_candidates = matches.map{|m| m.merge(confidence: confidences[m[:name]])}.sort_by {|m| -m[:score]}
-      HEITT::Logger.debug("Scored Algorithm: #{scored_candidates.map{|s| s[:name] }}  =>   Calculated Confidence: #{scored_candidates.map{|s| s[:confidence] }}")
       scored_candidates
     end
 
@@ -153,4 +150,4 @@ module HEITT
     end
   end
   #private_constant :Analyzer
-end
+end

data/lib/heitt/decoder.rb

@@ -0,0 +1,48 @@
+require 'base64'
+module HEITT
+  module Decoder
+    ENCODED_B64 = '[A-Za-z0-9+\/]'
+    BASE_HEXBYTE = '[[:xdigit:]]'
+
+    def self.binary?(string)
+      return true if string.encoding == Encoding::ASCII_8BIT && !string.valid_encoding?
+      return true if string.include?("\x00")
+      sample = string.bytes.first(4096)
+      non_printable = sample.count { |b| b < 32 || b > 126}
+      (non_printable.to_f / sample.size) > 0.30
+    end
+
+    def self.likely_base64?(string)
+      string.match(/\A(?:#{ENCODED_B64}{4})*(?:#{ENCODED_B64}{2}==|#{ENCODED_B64}{3}=)?\z/)
+    end
+
+    def self.likely_hex?(string)
+      string.match(/\A#{BASE_HEXBYTE}+\z/) && string.length.even?
+    end
+
+    def self.decode_layer(string)
+      return [:base64, Base64.decode64(string)] if likely_base64?(string)
+      return [:hex, [string].pack("H*")] if likely_hex?(string)
+      nil
+    end
+
+
+    def self.peel(string, peel_limit: 12)
+      steps = []
+      current = string.gsub(/\s+/, '')
+
+      peel_limit.times do
+        result = decode_layer(current)
+        break unless result
+        method, decoded = result
+        decoded = decoded.gsub(/\s+/, '')
+        break if binary?(decoded)
+
+        steps << { method: method, result: decoded }
+        current = decoded
+      end
+      { final: current, steps: steps }
+    end
+  end
+end
+          

data/lib/heitt/formatter.rb

@@ -43,6 +43,10 @@ module HEITT
             {text: "John Format: #{candidate[:john] || "--"}", children: []}
           ]
 
+          if extended
+            children << {text: "Extended: #{candidate[:extended]}", children: []}
+          end
+
           if verbose
             if candidate[:description] && !candidate[:description].empty?
               children << {text: "Description: #{candidate[:description]}", children: []}
@@ -86,7 +90,9 @@ module HEITT
                 hashcat: candidate[:hashcat],
                 john: candidate[:john],
                 confidence: candidate[:confidence],
-                description: candidate[:description]
+                extended: candidate[:extended],
+                description: candidate[:description],
+                notes: candidate[:notes]
               }
             end
           }
@@ -122,4 +128,4 @@ module HEITT
       result
     end
   end
-end
+end

data/lib/heitt/grouper.rb

@@ -1,4 +1,3 @@
-require_relative 'utils'
 module HEITT 
   module Grouper
 
@@ -15,8 +14,7 @@ module HEITT
           count: hashes.size
         }
       end
-      HEITT::Logger.debug("Hashes grouped successfully") unless groups.empty? || groups.nil?
       groups
     end
   end 
-end
+end

data/lib/heitt/scanner.rb

@@ -25,10 +25,7 @@ module HEITT
           matched = scanner.matched 
           next unless matched.length < 8 || HEITT::Analyzer.high_entropy?(matched, min_entropy)
           offset = scanner.pos - matched.length
-          HEITT::Logger.debug("Extracting prefix..")
           delim_prefix = HEITT::Analyzer.extract_prefix(text, offset)
-          HEITT::Logger.debug("Extracted prefix: #{delim_prefix.length <= 1 ? "NULL" : delim_prefix}")
-
           candidates = HEITT::Analyzer.score_candidates(modes, delim_prefix, context_scores)
           #score = candidates.first[:score]
 

data/lib/heitt/version.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 module HEITT
-  VERSION = "0.4.6"
-  GITHUB = "https://github.com/jobotow/heitt"
+  # version was changed to 0.50 on 30th Aug, 2026
+  VERSION = "0.5.0"
+  GITHUB = "https://codeberg.org/jobotow/heitt"
 end

data/lib/heitt.rb

@@ -3,8 +3,8 @@
 #require 'strscan'
 #require 'logger'
 #require 'colorize'
-require_relative 'heitt/utils'
 require_relative 'heitt/profiles'
+require_relative 'heitt/decoder'
 require_relative 'heitt/database'
 require_relative 'heitt/analyzer'
 require_relative 'heitt/scanner'
@@ -12,5 +12,7 @@ require_relative 'heitt/grouper'
 require_relative 'heitt/formatter'
 require_relative 'heitt/version'
 
-
-       
+# module HEITT
+# something like heitt = HEITT.new(database: nil, profiles: nil)
+# heitt.identify("hash_string")
+  

metadata

@@ -1,13 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: heitt
 version: !ruby/object:Gem::Version
-  version: 0.4.6
+  version: 0.5.0
 platform: ruby
 authors:
 - Jonathan Botchway Owusu
+autorequire:
 bindir: bin
 cert_chain: []
-date: 1980-01-02 00:00:00.000000000 Z
+date: 2026-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: colorize
@@ -35,18 +36,20 @@ files:
 - lib/heitt.rb
 - lib/heitt/analyzer.rb
 - lib/heitt/database.rb
+- lib/heitt/decoder.rb
 - lib/heitt/formatter.rb
 - lib/heitt/grouper.rb
 - lib/heitt/profiles.rb
 - lib/heitt/scanner.rb
 - lib/heitt/utils.rb
 - lib/heitt/version.rb
-homepage: https://github.com/jobotow/heitt
+homepage: https://codeberg.org/jobotow/heitt
 licenses: []
 metadata:
-  homepage_uri: https://github.com/jobotow/heitt
-  source_code_uri: https://github.com/jobotow/heitt
-  changelog_uri: https://github.com/jobotow/heitt/blob/main/CHANGELOG.md
+  homepage_uri: https://codeberg.org/jobotow/heitt
+  source_code_uri: https://codeberg.org/jobotow/heitt
+  changelog_uri: https://codeberg.org/jobotow/heitt/blob/main/CHANGELOG.md
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -61,7 +64,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.4.20
+signing_key:
 specification_version: 4
 summary: Hash Extraction, Identification and Triage Tool.
 test_files: []