sampling-hash 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MDAwYWQ2MTFmMWQ4ZTdlNDYxY2RhZDhjZGQ4M2MyODBjMTViMmJiOQ==
+  data.tar.gz: !binary |-
+    MTk2ZGJjOGYxYjIyM2QwZmI4ZGE1NWNiNjE3M2ZlMjcxYTU0YjVhNQ==
+!binary "U0hBNTEy":
+  metadata.gz: !binary |-
+    MjRjZDZjMGVjZTE1ZDczMDA1Y2NkYmJiMmY4NjkyODk0Y2I3MjdhMGU0ODFi
+    ODRkMmYzMTBiNDk5ZjVlMDVmZmIwY2M2MmY2Y2IzMWZmYzBiNjY5Mjg1NzBi
+    ZDlmZTJmMjg3YzllOGI1NTk0M2I5OTVkNGRkMTdmYzU5MTRlOTQ=
+  data.tar.gz: !binary |-
+    MjNmNThiZTRhNGQxNDMxOGJjOGMxZGJhYzBhNTE1NmY1MjkwNDg2NGIyYzk4
+    ZTg5YTAzNGJlYjVkY2Y1ZDE3YzdkMzZhNmM2OTBkZDMzZWJlYmE1NjYzNjI2
+    NjdlNDNjNWZkYWFlZTM5MjU2YjNmNzMzNzY1NjRhYTIxZDU1MTk=

data/lib/sampling-hash/sampling-io.rb

@@ -0,0 +1,42 @@
+module SamplingHash
+  class SamplingIO
+    def initialize(io)
+      raise ArgumentError, 'first parameter should be IO' unless io.kind_of?(IO)
+
+      @io = io
+      @chunk = 0
+    end
+
+    def sample
+      return nil if @chunk > samples
+
+      @io.seek(offset, IO::SEEK_SET)
+      @chunk += 1
+      @io.read(CHUNK_SIZE)
+    end
+
+  private
+
+    CHUNK_SIZE = 256
+
+    def file_size
+      @file_size ||= @io.stat.size
+    end
+
+    def reduce
+      (Math.log(file_size / 1000) * 1000).truncate & 0xFFFFFF00
+    end
+
+    def samples_size
+      @samples_size ||= (file_size < 3000 ? file_size : reduce)
+    end
+
+    def samples
+      samples_size / CHUNK_SIZE + 1
+    end
+
+    def offset
+      @chunk * CHUNK_SIZE
+    end
+  end
+end

data/lib/sampling-hash/version.rb

@@ -0,0 +1,3 @@
+module SamplingHash
+  VERSION = '0.0.1'
+end

data/lib/sampling-hash.rb

@@ -0,0 +1,18 @@
+require 'sampling-hash/sampling-io'
+require 'sampling-hash/version'
+require 'xxhash'
+
+module SamplingHash
+  def self.hash(path, seed)
+    raise ArgumentError, 'file not found' unless File.file?(path)
+
+    hash = XXhash::Internal::StreamingHash.new(seed)
+    sio = SamplingIO.new(File.open(path, 'r'))
+
+    while chunk = sio.sample
+      hash.update(chunk)
+    end
+
+    hash.digest
+  end
+end

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: sampling-hash
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Malte Rohde
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-04 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: xxhash
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: 0.2.0
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Calculates deterministic hashes from file samples
+email:
+- malte.rohde@flavoursys.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/sampling-hash.rb
+- lib/sampling-hash/version.rb
+- lib/sampling-hash/sampling-io.rb
+homepage: http://github.com/flavoursys/sampling-hash
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.0.3
+signing_key: 
+specification_version: 4
+summary: Sampling hash algorithm for large files
+test_files: []