sampling-hash 0.0.1 → 0.1.2

checksums.yaml

@@ -1,15 +1,7 @@
 ---
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    MDAwYWQ2MTFmMWQ4ZTdlNDYxY2RhZDhjZGQ4M2MyODBjMTViMmJiOQ==
-  data.tar.gz: !binary |-
-    MTk2ZGJjOGYxYjIyM2QwZmI4ZGE1NWNiNjE3M2ZlMjcxYTU0YjVhNQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    MjRjZDZjMGVjZTE1ZDczMDA1Y2NkYmJiMmY4NjkyODk0Y2I3MjdhMGU0ODFi
-    ODRkMmYzMTBiNDk5ZjVlMDVmZmIwY2M2MmY2Y2IzMWZmYzBiNjY5Mjg1NzBi
-    ZDlmZTJmMjg3YzllOGI1NTk0M2I5OTVkNGRkMTdmYzU5MTRlOTQ=
-  data.tar.gz: !binary |-
-    MjNmNThiZTRhNGQxNDMxOGJjOGMxZGJhYzBhNTE1NmY1MjkwNDg2NGIyYzk4
-    ZTg5YTAzNGJlYjVkY2Y1ZDE3YzdkMzZhNmM2OTBkZDMzZWJlYmE1NjYzNjI2
-    NjdlNDNjNWZkYWFlZTM5MjU2YjNmNzMzNzY1NjRhYTIxZDU1MTk=
+SHA256:
+  metadata.gz: e944cda00228ac5a7405198955de90e17134f243b828daead2fab77155103030
+  data.tar.gz: de8b6e76b09460cfad680c842edbe195922e9db5a48de1fde050be2d30156d47
+SHA512:
+  metadata.gz: a0a7043b089963ecdff6986b9026aef765cb12cfe0630283b8c6719385c80b7531650c8a4875b8923025b3b239ec8641d9d9bb180be971e39420f6242614402c
+  data.tar.gz: '0295e1b2f327a286b3bc1029dc15fd141d92f2a3c424404f9ea9534b71dfc45a3bf1d893b9c34edfc62abbd013309905b8bd460a4ec0d8b06ff3058e622db26c'

data/lib/sampling-hash.rb

@@ -1,18 +1,31 @@
+require 'sampling-hash/hash'
+require 'sampling-hash/sampler'
 require 'sampling-hash/sampling-io'
 require 'sampling-hash/version'
 require 'xxhash'
 
 module SamplingHash
-  def self.hash(path, seed)
+  # We default to 64 bit xxhash.
+  def self.hash(path, seed = File.size(path), hash = XXhash::XXhashInternal::StreamingHash64.new(seed))
     raise ArgumentError, 'file not found' unless File.file?(path)
 
-    hash = XXhash::Internal::StreamingHash.new(seed)
-    sio = SamplingIO.new(File.open(path, 'r'))
+    File.open(path, 'r') do |fd|
+
+      sio = SamplingIO.new(fd)
+      sio.samples do |chunk|
+        hash.update(chunk)
+      end
+
+      hash.digest
 
-    while chunk = sio.sample
-      hash.update(chunk)
     end
+  end
+
+  def self.hash32(path, seed = File.size(path))
+    hash path, seed, XXHash::XXhashInternal::StreamingHash32.new(seed)
+  end
 
-    hash.digest
+  def self.hash64(path, seed = File.size(path))
+    hash path, seed, XXHash::XXhashInternal::StreamingHash64.new(seed)
   end
 end

data/lib/sampling-hash/hash.rb

@@ -0,0 +1,97 @@
+module SamplingHash
+  class Hash
+    def initialize(size, seed = size, sampler = nil, xxhash = XXhash::XXhashInternal::StreamingHash64.new(seed))
+      @sampler = sampler || Sampler.new(size)
+      @xxhash = xxhash
+      
+      # Position in data stream.
+      @position = 0
+
+      # Current sample.
+      @current_sample        = nil # The data.
+      @current_sample_offset = 0   # The offset (within the stream).
+      @current_sample_size   = 0   # The sample size.
+      @next                  = 0   # The next sample index.
+
+      # Start.
+      next_sample
+    end
+
+    def update(chunk)
+      pos = 0
+      while pos < chunk.size
+        len = chunk.size - pos
+        used = advance(chunk, pos, len)
+        @position += used
+        pos += used
+      end
+    end
+
+    def digest
+      @xxhash.digest
+    end
+
+  private
+
+    def advance(chunk, pos, len)
+      if in_sample?
+        # Use some bytes.
+        msb = missing_sample_bytes
+        if msb > len
+          update_sample chunk[pos..(pos + len - 1)]
+          len
+        else
+          finish_sample chunk[pos..(pos + msb - 1)]
+          msb
+        end
+      elsif samples_left?
+        # Discard some bytes until the next sample starts.
+        mgb = missing_gap_bytes
+        if mgb > len
+          len
+        else
+          mgb
+        end
+      else
+        # Discard the rest.
+        len
+      end
+    end
+
+    def in_sample?
+      samples_left? && @position >= @current_sample_offset && @position < @current_sample_offset + @current_sample_size
+    end
+
+    def samples_left?
+      !!@current_sample
+    end
+
+    def missing_sample_bytes
+      @current_sample_size - @current_sample.size
+    end
+
+    def missing_gap_bytes
+      @current_sample_offset - @position
+    end
+
+    def update_sample(data)
+      @current_sample += data
+    end
+
+    def finish_sample(data)
+      @current_sample += data
+      @xxhash.update(@current_sample)
+      next_sample
+    end
+
+    def next_sample
+      if @next < @sampler.samples.size
+        @current_sample = String.new
+        @current_sample_offset, @current_sample_size = @sampler.samples[@next]
+        @next += 1
+      else
+        @current_sample = nil
+      end
+    end
+  end
+end

data/lib/sampling-hash/sampler.rb

@@ -0,0 +1,60 @@
+module SamplingHash
+  class Sampler
+    include Enumerable
+
+    attr_reader :samples, :size
+
+    # Calculates sample offsets.
+    # 
+    # Parameters:
+    # - sample_size: Size of a sample (in bytes).
+    # - header_samples: Number of samples at front of data always to be included.
+    # - minimum_samples: Minimum number of samples to be included.
+    # - remaining_factor: If size is greater than minimum_samples * sample_size, this specifies the
+    #              linear factor function used to determine the additional data used.
+    def initialize(size, sample_size = 1024, header_samples = 1000, minimum_samples = 5000, remaining_factor = 0.001)
+      @samples = []
+
+      minimum_sampling_size = minimum_samples * sample_size
+      if (size > minimum_sampling_size)
+        # Continuous header samples first.
+        header_samples.times { |i| @samples << [i * sample_size, sample_size] }
+
+        # Spread the rest.
+        start_offset                       = header_samples * sample_size
+        remaining_size                     = size - start_offset
+
+        remaining_minimum_samples          = [0, minimum_samples - header_samples].max
+        remaining_minimum_sampling_size    = remaining_minimum_samples * sample_size
+
+        remaining_additional_size          = remaining_size - remaining_minimum_sampling_size
+        remaining_additional_sampling_size = remaining_additional_size * remaining_factor
+        remaining_additional_samples       = (remaining_additional_sampling_size / sample_size).truncate
+
+        remaining_total_samples            = remaining_minimum_samples + remaining_additional_samples
+        remaining_total_sampling_size      = remaining_minimum_sampling_size + remaining_additional_sampling_size
+
+        remaining_unsampled_size           = remaining_size - remaining_total_sampling_size
+        remaining_sampling_gap             = (remaining_unsampled_size / remaining_total_samples).truncate
+
+        # NOTE: We can not overflow since we calculated the remaining_additional_samples with integer division.
+        remaining_total_samples.times do |i|
+          @samples << [start_offset + i * (sample_size + remaining_sampling_gap), sample_size]
+        end
+      else
+        total_full_samples = size / sample_size
+        last_sample_size   = size - ((size / sample_size) * sample_size)
+
+        # Simply take them all.
+        total_full_samples.times { |i| @samples << [i * sample_size, sample_size] }
+        @samples << [total_full_samples * sample_size, last_sample_size] if last_sample_size != 0
+      end
+
+      @size = @samples.inject(0) { |i, v| i + v[1] }
+    end
+
+    def each(&block)
+      @samples.each(&block)
+    end
+  end
+end

data/lib/sampling-hash/sampling-io.rb

@@ -1,42 +1,17 @@
 module SamplingHash
   class SamplingIO
-    def initialize(io)
+    def initialize(io, sampler = nil)
       raise ArgumentError, 'first parameter should be IO' unless io.kind_of?(IO)
 
       @io = io
-      @chunk = 0
-    end
-
-    def sample
-      return nil if @chunk > samples
-
-      @io.seek(offset, IO::SEEK_SET)
-      @chunk += 1
-      @io.read(CHUNK_SIZE)
-    end
-
-  private
-
-    CHUNK_SIZE = 256
-
-    def file_size
-      @file_size ||= @io.stat.size
-    end
-
-    def reduce
-      (Math.log(file_size / 1000) * 1000).truncate & 0xFFFFFF00
-    end
-
-    def samples_size
-      @samples_size ||= (file_size < 3000 ? file_size : reduce)
+      @sampler = sampler || Sampler.new(io.stat.size)
     end
 
     def samples
-      samples_size / CHUNK_SIZE + 1
-    end
-
-    def offset
-      @chunk * CHUNK_SIZE
+      @sampler.each do |offset, size|
+        @io.seek(offset, IO::SEEK_SET)
+        yield @io.read(size)
+      end
     end
   end
 end

data/lib/sampling-hash/version.rb

@@ -1,3 +1,3 @@
 module SamplingHash
-  VERSION = '0.0.1'
+  VERSION = '0.1.2'
 end

metadata

@@ -1,69 +1,71 @@
 --- !ruby/object:Gem::Specification
 name: sampling-hash
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.1.2
 platform: ruby
 authors:
-- Malte Rohde
+- Projective Technology GmbH
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-09-04 00:00:00.000000000 Z
+date: 2021-04-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: xxhash
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.3'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.2.0
+        version: '0.3'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.3.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 12.3.3
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.5'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ! '>='
+    - - "~>"
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '5.5'
 description: Calculates deterministic hashes from file samples
-email:
-- malte.rohde@flavoursys.com
+email: technology@projective.io
 executables: []
 extensions: []
 extra_rdoc_files: []
 files:
 - lib/sampling-hash.rb
-- lib/sampling-hash/version.rb
+- lib/sampling-hash/hash.rb
+- lib/sampling-hash/sampler.rb
 - lib/sampling-hash/sampling-io.rb
-homepage: http://github.com/flavoursys/sampling-hash
-licenses: []
+- lib/sampling-hash/version.rb
+homepage: https://github.com/projectivetech/sampling-hash
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -71,17 +73,16 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 3.2.5
 signing_key: 
 specification_version: 4
 summary: Sampling hash algorithm for large files