minwise 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: 24510ece26235c8ef8956326e2d05d9162b70b4bede324ab6e548d2746194b8a
+  data.tar.gz: b1606b461d02f4a420daf8e326ce75e76af5ce7cf0bbf43f7064247c0ea0b968
+SHA512:
+  metadata.gz: 7867b8181456107e42783cb7d1e2a62827710e8bfd5e9f4cd94d9521fbee069841c19b8a636d5dba477c4211d4bd2134d4cad1bdfcc9322e714853d696250959
+  data.tar.gz: e43b8b14d8cc308ce0b752c8de667d163f29905398b0201cf255b5990e53b5b5d1d67f75b3596a752631bc48c192b0fde3413a7a73b5f3f4dbef56fbd70e343c

data/.rubocop.yml

@@ -0,0 +1,18 @@
+require:
+  - rubocop-rake
+  - rubocop-minitest
+
+AllCops:
+  TargetRubyVersion: 2.6
+  NewCops: enable
+
+Style/StringLiterals:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Style/StringLiteralsInInterpolation:
+  Enabled: true
+  EnforcedStyle: double_quotes
+
+Layout/LineLength:
+  Max: 120

data/CHANGELOG.md

@@ -0,0 +1,5 @@
+## [Unreleased]
+
+## [0.1.0] - 2023-08-14
+
+- Initial release

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2023 Samuel Scully
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,50 @@
+# Minwise
+
+Fast locality sensitive hashes using the minhash algorithm.
+
+## Installation
+
+Install the gem and add to the application's Gemfile by executing:
+
+    $ bundle add minwise
+
+If bundler is not being used to manage dependencies, install the gem by executing:
+
+    $ gem install minwise
+
+## Usage
+
+Generate the minhash of a string:
+
+```ruby
+Minwise::Minhash.digest("Chunky bacon")
+# => [437974493, 147728091, 1185236492, ...]
+```
+
+Generate a minhash with options:
+
+```ruby
+Minwise::Minhash.digest("Chunky bacon", shingle_size: 9, hash_size: 500, seed: 42)
+# => [203094719, 599941115, 1256960069, ...]
+```
+
+You can also generate a minhash of a bare set of integers:
+
+```ruby
+Minwise::Minhash.digest([1, 2, 3])
+# => [1005141192, 713750329, 346603495, ...]
+```
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake test` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/sbscully/minwise.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

data/Rakefile

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "bundler/gem_tasks"
+require "rake/testtask"
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << "test"
+  t.libs << "lib"
+  t.test_files = FileList["test/**/test_*.rb"]
+end
+
+require "rubocop/rake_task"
+
+RuboCop::RakeTask.new
+
+require "rake/extensiontask"
+
+desc "Compile all the extensions"
+task build: :compile
+
+Rake::ExtensionTask.new("minwise") do |ext|
+  ext.lib_dir = "lib/minwise"
+end
+
+desc "Benchmark Minwise::Minhash.digest"
+task benchmark: :build do
+  ruby "./test/benchmark.rb"
+end
+
+task default: %i[clobber compile test rubocop]

data/ext/minwise/extconf.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+# Makes all symbols private by default to avoid unintended conflict
+# with other gems. To explicitly export symbols you can use RUBY_FUNC_EXPORTED
+# selectively, or entirely remove this flag.
+append_cflags("-fvisibility=hidden")
+
+create_makefile("minwise/minwise")

data/ext/minwise/minwise.c

@@ -0,0 +1,115 @@
+#include <stdint.h>
+#include "minwise.h"
+
+static VALUE mMinwise;
+static VALUE cMinwiseMinhash;
+
+static uint64_t randr(uint64_t min, uint64_t max, uint64_t seed) {
+  // xorshift*
+  uint64_t x = seed + 1; // must not be zero
+  x ^= x >> 12;
+  x ^= x << 25;
+  x ^= x >> 27;
+  x *= 0x2545F4914F6CDD1DULL;
+
+  return (uint64_t)(min + (double)x / UINT64_MAX * (max - min));
+}
+
+static void minhash(const uint32_t* set, const size_t set_len, uint32_t* hash, const size_t hash_len, uint64_t seed) {
+  const uint64_t p = 4294967311; // first prime greater than UINT32_MAX
+  uint64_t a, b;
+  uint32_t x, h;
+
+  for (size_t i = 0; i < hash_len; i++) {
+    a = randr(0, p, seed + i);
+    a = (a / 2) * 2 - 1; // must be odd
+    b = randr(0, p, a);
+
+    h = UINT32_MAX;
+    for (size_t j = 0; j < set_len; j++) {
+      x = (uint32_t)(((a * set[j] + b) % p) % UINT32_MAX);
+
+      if (x < h) {
+        h = x;
+      }
+    }
+
+    hash[i] = h;
+  }
+}
+
+static VALUE c_minhash(VALUE self, VALUE rb_set, VALUE rb_hash_len, VALUE rb_hash_seed) {
+  Check_Type(rb_set, T_ARRAY);
+
+  size_t rb_set_len = RARRAY_LEN(rb_set);
+  size_t c_hash_len = NUM2SIZET(rb_hash_len);
+  uint64_t c_hash_seed = NUM2ULONG(rb_hash_seed);
+
+  uint32_t *c_set = (uint32_t *)malloc(rb_set_len * sizeof(uint32_t));
+  for (size_t i = 0; i < rb_set_len; i++) {
+    c_set[i] = NUM2UINT(rb_ary_entry(rb_set, i));
+  }
+
+  uint32_t *c_hash = (uint32_t *)malloc(c_hash_len * sizeof(uint32_t));
+  minhash(c_set, rb_set_len, c_hash, c_hash_len, c_hash_seed);
+
+  VALUE rb_hash = rb_ary_new_capa(c_hash_len);
+  for (size_t i = 0; i < c_hash_len; i++) {
+    rb_ary_store(rb_hash, i, UINT2NUM(c_hash[i]));
+  }
+
+  free(c_set);
+  free(c_hash);
+
+  return rb_hash;
+}
+
+static uint32_t fnv1a(char *str) {
+  uint32_t hash = 0x811c9dc5;
+  unsigned char *s = (unsigned char *)str;
+
+  while (*s) {
+    hash ^= (uint32_t)*s++;
+    hash *= 0x01000193;
+  }
+
+  return hash;
+}
+
+static VALUE c_tokenize(VALUE self, VALUE rb_string, VALUE rb_shingle_size) {
+  char *c_string = StringValueCStr(rb_string);
+  size_t c_string_len = RSTRING_LEN(rb_string);
+  size_t c_shingle_size = NUM2SIZET(rb_shingle_size);
+
+  if (c_string_len <= c_shingle_size) {
+    VALUE rb_tokens = rb_ary_new_capa(1);
+    rb_ary_store(rb_tokens, 0, UINT2NUM(fnv1a(c_string)));
+
+    return rb_tokens;
+  }
+
+  size_t rb_tokens_len = c_string_len - c_shingle_size + 1;
+  VALUE rb_tokens = rb_ary_new_capa(rb_tokens_len);
+
+  char buffer[c_shingle_size + 1];
+  for (size_t i = 0; i < rb_tokens_len; i++) {
+    for (size_t j = 0; j < c_shingle_size; j++) {
+      buffer[j] = c_string[i + j];
+    }
+    buffer[c_shingle_size + 1] = 0;
+
+    rb_ary_store(rb_tokens, i, UINT2NUM(fnv1a(buffer)));
+  }
+
+  return rb_tokens;
+}
+
+RUBY_FUNC_EXPORTED void
+Init_minwise(void)
+{
+  mMinwise = rb_define_module("Minwise");
+  cMinwiseMinhash = rb_define_class_under(mMinwise, "Minhash", rb_cObject);
+
+  rb_define_singleton_method(cMinwiseMinhash, "__hash", c_minhash, 3);
+  rb_define_singleton_method(cMinwiseMinhash, "__tokenize", c_tokenize, 2);
+}

data/ext/minwise/minwise.h

@@ -0,0 +1,6 @@
+#ifndef MINWISE_H
+#define MINWISE_H 1
+
+#include "ruby.h"
+
+#endif /* MINWISE_H */

data/lib/minwise/minhash.rb

@@ -0,0 +1,80 @@
+# frozen_string_literal: true
+
+module Minwise
+  # The classic minhash algorithm.
+  #
+  # The minhash digests of two arrays will share approximately the same
+  # proportion of elements as the underlying arrays. This is useful for finding
+  # similar items in large datasets.
+  #
+  # The interface is similar to classes in the `Digest` module of the standard
+  # library.
+  #
+  #   Minwise::Minhash.digest([1, 2, 3])
+  #   # => [1005141192, 713750329, 346603495, ...]
+  #
+  # String inputs are first converted to an array of fixed width chunks of
+  # characters called "shingles". For example `"Chunky"` with a shingle
+  # size of 3 would become `["Chu", "hun", "unk", "nky"]`, then that array is
+  # used to generate the minhash. The `shingle_size` option controls the chunk
+  # width used.
+  #
+  #   Minwise::Minhash.digest("Chunky bacon", shingle_size: 3)
+  #   # => [437974493, 147728091, 1185236492, ...]
+  #
+  # The size of the output `hash_size` and `seed` for generating the hash can
+  # also be set in options. Larger minhashes will give more accurate estimates
+  # of similarity between items, but are slower to generate and take more space.
+  #
+  #   Minwise::Minhash.digest("Chunky bacon", hash_size: 900, seed: 84)
+  #   # => [355390344, 825885127, 262059926, ...]
+  #
+  # When comparing two minhashes all the options used to generate the minhashes
+  # must be identical for the comparison to be meaningful.
+  #
+  # Detailed information on how the minhash algorithm works and how minhashes
+  # can be used can be found in "Chapter 3: Finding Similar Items" of the book
+  # "Mining of Massive Datasets", by Leskovec, Rajaraman, and Ulman, available
+  # for free at http://www.mmds.org/.
+  #
+  class Minhash
+    DEFAULT_OPTIONS = {
+      hash_size: 128,
+      shingle_size: 5,
+      seed: 3_141_592
+    }.freeze
+
+    def self.digest(data, options = {})
+      new(data, options).digest
+    end
+
+    def initialize(data = [], options = {})
+      @options = DEFAULT_OPTIONS.merge(options)
+      @data = parse(data)
+    end
+
+    def update(element)
+      @data << element
+    end
+
+    def digest
+      raise ArgumentError, "input must not be empty" if @data.empty?
+
+      self.class.__hash(@data, @options[:hash_size], @options[:seed])
+    end
+
+    private
+
+    def parse(data)
+      return [] if data.empty?
+
+      if data.respond_to?(:to_a)
+        data
+      elsif data.respond_to?(:to_str)
+        self.class.__tokenize(data.to_str, @options[:shingle_size])
+      else
+        raise ArgumentError, "input must be a string or array of integers"
+      end
+    end
+  end
+end

data/lib/minwise/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Minwise
+  VERSION = "0.1.0"
+end

data/lib/minwise.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative "minwise/version"
+require_relative "minwise/minwise"
+require_relative "minwise/minhash"
+
+# A Ruby library for generating minwise hashes.
+module Minwise
+  class Error < StandardError; end
+
+  class << self
+    # Returns the Jaccard similarity of 2 arrays, a number between 0.0 and 1.0.
+    #
+    # The arrays are treated as sets, i.e. duplicate elements in an array are
+    # only counted once.
+    def similarity(set_one, set_two)
+      set_one.intersection(set_two).length / set_one.union(set_two).length.to_f
+    end
+  end
+end

data/sig/minwise.rbs

@@ -0,0 +1,4 @@
+module Minwise
+  VERSION: String
+  # See the writing guide of rbs: https://github.com/ruby/rbs#guides
+end

metadata

@@ -0,0 +1,60 @@
+--- !ruby/object:Gem::Specification
+name: minwise
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Samuel Scully
+autorequire:
+bindir: exe
+cert_chain: []
+date: 2023-08-17 00:00:00.000000000 Z
+dependencies: []
+description:
+email:
+- sbscully@gmail.com
+executables: []
+extensions:
+- ext/minwise/extconf.rb
+extra_rdoc_files: []
+files:
+- ".rubocop.yml"
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- Rakefile
+- ext/minwise/extconf.rb
+- ext/minwise/minwise.c
+- ext/minwise/minwise.h
+- lib/minwise.rb
+- lib/minwise/minhash.rb
+- lib/minwise/version.rb
+- sig/minwise.rbs
+homepage: https://github.com/sbscully/minwise
+licenses:
+- MIT
+metadata:
+  rubygems_mfa_required: 'true'
+  homepage_uri: https://github.com/sbscully/minwise
+  source_code_uri: https://github.com/sbscully/minwise
+  changelog_uri: https://github.com/sbscully/minwise/blob/main/CHANGELOG.md
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.3.7
+signing_key:
+specification_version: 4
+summary: Fast locality sensitive hashes using minwise hashing and derivatives.
+test_files: []