tlsh 0.1.1

data/lib/tlsh/quartiles.rb

@@ -0,0 +1,137 @@
+# Buckets provides utility computation methods for computation of quartile statistics
+module Quartiles
+  class << self
+    EFF_BUCKETS = 128
+
+    def quartile_points(buckets)
+      spl = spr = 0
+      q1 = 0
+
+      p1 = EFF_BUCKETS / 4 - 1
+      p2 = EFF_BUCKETS / 2 - 1
+      p_end = EFF_BUCKETS - 1
+
+      buckets_cpy = buckets.dup[0..EFF_BUCKETS]
+
+      cut_left = []
+      cut_right = []
+
+      l = 0
+      r = p_end
+      loop do
+        ret = partition(buckets_cpy, l, r)
+        if ret > p2
+          r = ret - 1
+          cut_right[spr] = ret
+          spr += 1
+        elsif ret < p2
+          l = ret + 1
+          cut_left[spl] = ret
+          spl += 1
+        else
+          q1 = buckets_cpy[p2]
+          break
+        end
+      end
+
+      cut_left[spl] = p2 - 1
+      cut_right[spr] = p2 + 1
+
+      q2 = get_q2(buckets_cpy, cut_left, spl, p1)
+      q3 = get_q3(buckets_cpy, cut_right, spr, p_end)
+
+      [q1, q2, q3]
+    end
+
+    private
+
+    def partition(buffer, left, right)
+      return left if left == right
+
+      if left + 1 == right
+        if buffer[left] > buffer[right]
+          buffer[right], buffer[left] = buffer[left], buffer[right]
+        end
+        return left
+      end
+
+      ret = left
+
+      partition_buffer(buffer, ret, left, right)
+    end
+
+    def partition_buffer(buffer, ret, left, right)
+      pivot = (left + right) >> 1
+      value = buffer[pivot]
+
+      buffer[pivot] = buffer[right]
+      buffer[right] = value
+
+      (left..right).each do |i|
+        if buffer[i] < value
+          buffer[i], buffer[ret] = buffer[ret], buffer[i]
+          ret += 1
+        end
+
+        buffer[right] = buffer[ret]
+        buffer[ret] = value
+      end
+
+      ret
+    end
+
+    def get_q2(buckets, cut_left, spl, p1)
+      i = l = 0
+      while i <= spl
+        r = cut_left[i]
+
+        if r > p1
+          loop do
+            ret = partition(buckets, l, r)
+            if ret > p1
+              r = ret - 1
+            elsif ret < p1
+              l = ret + 1
+            else
+              return buckets[p1]
+            end
+          end
+        end
+
+        i += 1
+      end
+    end
+
+    def get_q3(buckets, cut_right, spr, p_end)
+      p3 = EFF_BUCKETS - EFF_BUCKETS / 4 - 1
+      q3 = 0
+
+      i = 0
+      r = p_end
+      while i <= spr
+        l = cut_right[i]
+        if l < p3
+          loop do
+            ret = partition(buckets, l, r)
+            if ret > p3
+              r = ret - 1
+            elsif ret < p3
+              l = ret + 1
+            else
+              q3 = buckets[p3]
+              break
+            end
+          end
+          break
+        elsif l > p3
+          r = l
+        else
+          q3 = buckets[p3]
+        end
+
+        i += 1
+      end
+      q3
+    end
+  end
+end

data/lib/tlsh/tlsh.rb

@@ -0,0 +1,74 @@
+require 'tlsh/version'
+require 'tlsh/distance/distance'
+require 'tlsh/digest_hash/pearson'
+
+# Tlsh module implement interface for TLSH (Trend Micro Locality Sensitive Hash) computation.
+# TLSH is usable for diff and similarity computations of binary data, because of the locality sensitivity.
+module Tlsh
+  LOG1_5 = 0.4054651
+  LOG1_3 = 0.26236426
+  LOG1_1 = 0.095310180
+
+  class << self
+    def diff_files(filename, other_filename)
+      file_a = File.read(filename)
+      file_b = File.read(other_filename)
+
+      tslh_a = tlsh_hash(file_a.bytes)
+      tslh_b = tlsh_hash(file_b.bytes)
+      tslh_a.diff(tslh_b)
+    end
+
+    # hash_file calculates the TLSH for the input file
+    def hash_file(filename)
+      file = File.read(filename)
+      tlsh_hash(file.bytes)
+    end
+
+    def hash_bytes(blob)
+      tlsh_hash(blob)
+    end
+
+    private
+
+    def tlsh_hash(input)
+      buckets, checksum, filesize = Buckets.fill_buckets(input)
+
+      # get the quartiles and their ratio
+      q1, q2, q3 = Quartiles.quartile_points(buckets)
+      q1_ratio = (q1 * 100 / q3) % 16
+      q2_ratio = (q2 * 100 / q3) % 16
+      q_ratio = ((q1_ratio & 0xF) << 4) | (q2_ratio & 0xF)
+
+      # get the binary buckets representation
+      bin_hash = Buckets.buckets_binary(buckets, q1, q2, q3)
+
+      TlshInstance.new(checksum: checksum, l_value: l_value(filesize), q1_ratio: q1_ratio, q2_ratio: q2_ratio, q_ratio: q_ratio, body: bin_hash)
+    end
+
+    def l_value(length)
+      l = if length <= 656
+            l_value_small(length)
+
+          elsif length <= 3199
+            l_value_medium(length)
+
+          else
+            l_value_large(length)
+          end
+      l & 255
+    end
+
+    def l_value_small(length)
+      Float(Math.log(length) / LOG1_5).floor.to_i
+    end
+
+    def l_value_medium(length)
+      Float(Math.log(length) / LOG1_3 - 8.72777).floor.to_i
+    end
+
+    def l_value_large(length)
+      Float(Math.log(length) / LOG1_1 - 62.5472).floor.to_i
+    end
+  end
+end

data/lib/tlsh/tlsh_instance.rb

@@ -0,0 +1,39 @@
+module Tlsh
+  # TlshInstance represents single TLSH instance
+  class TlshInstance
+    attr_accessor :checksum, :l_value, :q1_ratio, :q2_ratio, :q_ratio, :body
+
+    def initialize(params = {})
+      params.each do |key, value|
+        setter = "#{key}="
+        send(setter, value) if respond_to?(setter.to_sym, false)
+      end
+    end
+
+    # returns diff against another TlshInstance. The closer to 0, the smaller the diff.
+    def diff(other)
+      Distance.diff_total(self, other, true)
+    end
+
+    # returns the binary representation of the hash
+    def binary
+      [swap_byte(checksum), swap_byte(l_value), q_ratio] + body
+    end
+
+    # returns the string representation of the hash
+    def string
+      binary.map { |i| i.to_i.to_s(16) }.join('')
+    end
+
+    def comparable?
+      checksum && l_value && q1_ratio && q2_ratio && q_ratio && body
+    end
+
+    private
+
+    def swap_byte(input)
+      out = ((input & 0xF0) >> 4) & 0x0F
+      out | ((input & 0x0F) << 4) & 0xF0
+    end
+  end
+end

data/lib/tlsh/version.rb

@@ -0,0 +1,3 @@
+module Tlsh
+  VERSION = '0.1.1'.freeze
+end

data/tlsh.gemspec

@@ -0,0 +1,41 @@
+# coding: utf-8
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'tlsh/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = 'tlsh'
+  spec.version       = Tlsh::VERSION
+  spec.authors       = ['adamliesko']
+  spec.email         = ['adamliesko@gmail.com']
+
+  spec.summary       = 'A fuzzy matching library which creates hashes that can be used for similarity comparisons.'
+  spec.description = <<DESC
+tlsh is a fuzzy matching library, which hashes can be used for similarity comparison.
+Given a byte stream with a minimum length of 256 bytes, TLSH generates a hash value
+which can be used for similarity comparisons. Similar objects will have similar hash
+values which allow for the detection of similar objects by comparing their hash values.
+
+The computed hash is 35 bytes long (output as 70 hexadecimal characters).
+The first 3 bytes are used to capture the information about the file as a whole (length, ...),
+while the last 32 bytes are used to capture information about incremental parts of the file.
+DESC
+
+  spec.homepage      = 'https://github.com/adamliesko/tlsh'
+  spec.license       = 'MIT'
+
+  spec.files = `git ls-files -z`.split("\x0").reject do |f|
+    f.match(%r{^(test|spec|features)/})
+  end
+
+  spec.bindir        = 'exe'
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ['lib']
+
+  spec.add_development_dependency 'bundler', '~> 1.15'
+  spec.add_development_dependency 'rake', '~> 10.0'
+  spec.add_development_dependency 'minitest', '~> 5.0'
+  spec.add_development_dependency 'coveralls', '~> 0'
+  spec.add_development_dependency 'simplecov', '~> 0'
+end

metadata

@@ -0,0 +1,143 @@
+--- !ruby/object:Gem::Specification
+name: tlsh
+version: !ruby/object:Gem::Version
+  version: 0.1.1
+platform: ruby
+authors:
+- adamliesko
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2017-08-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.15'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '5.0'
+- !ruby/object:Gem::Dependency
+  name: coveralls
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: simplecov
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: |
+  tlsh is a fuzzy matching library, which hashes can be used for similarity comparison.
+  Given a byte stream with a minimum length of 256 bytes, TLSH generates a hash value
+  which can be used for similarity comparisons. Similar objects will have similar hash
+  values which allow for the detection of similar objects by comparing their hash values.
+
+  The computed hash is 35 bytes long (output as 70 hexadecimal characters).
+  The first 3 bytes are used to capture the information about the file as a whole (length, ...),
+  while the last 32 bytes are used to capture information about incremental parts of the file.
+email:
+- adamliesko@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- ".rubocop.yml"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- lib/tlsh.rb
+- lib/tlsh/buckets.rb
+- lib/tlsh/digest_hash/pearson.rb
+- lib/tlsh/distance/distance.rb
+- lib/tlsh/distance/precomputed_bits.rb
+- lib/tlsh/quartiles.rb
+- lib/tlsh/tlsh.rb
+- lib/tlsh/tlsh_instance.rb
+- lib/tlsh/version.rb
+- tlsh.gemspec
+homepage: https://github.com/adamliesko/tlsh
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.6.12
+signing_key: 
+specification_version: 4
+summary: A fuzzy matching library which creates hashes that can be used for similarity
+  comparisons.
+test_files: []