uncle_blake3 0.0.1

data/ext/blake3/c/example.c

@@ -0,0 +1,37 @@
+#include "blake3.h"
+#include <errno.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+int main() {
+  // Initialize the hasher.
+  blake3_hasher hasher;
+  blake3_hasher_init(&hasher);
+
+  // Read input bytes from stdin.
+  unsigned char buf[65536];
+  while (1) {
+    ssize_t n = read(STDIN_FILENO, buf, sizeof(buf));
+    if (n > 0) {
+      blake3_hasher_update(&hasher, buf, n);
+    } else if (n == 0) {
+      break; // end of file
+    } else {
+      fprintf(stderr, "read failed: %s\n", strerror(errno));
+      exit(1);
+    }
+  }
+
+  // Finalize the hash. BLAKE3_OUT_LEN is the default output length, 32 bytes.
+  uint8_t output[BLAKE3_OUT_LEN];
+  blake3_hasher_finalize(&hasher, output, BLAKE3_OUT_LEN);
+
+  // Print the hash as hexadecimal.
+  for (size_t i = 0; i < BLAKE3_OUT_LEN; i++) {
+    printf("%02x", output[i]);
+  }
+  printf("\n");
+  return 0;
+}

data/ext/blake3/c/main.c

@@ -0,0 +1,166 @@
+/*
+ * This main file is intended for testing via `make test`. It does not build in
+ * other settings. See README.md in this directory for examples of how to build
+ * C code.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#include "blake3.h"
+#include "blake3_impl.h"
+
+#define HASH_MODE 0
+#define KEYED_HASH_MODE 1
+#define DERIVE_KEY_MODE 2
+
+static void hex_char_value(uint8_t c, uint8_t *value, bool *valid) {
+  if ('0' <= c && c <= '9') {
+    *value = c - '0';
+    *valid = true;
+  } else if ('a' <= c && c <= 'f') {
+    *value = 10 + c - 'a';
+    *valid = true;
+  } else {
+    *valid = false;
+  }
+}
+
+static int parse_key(char *hex_key, uint8_t out[BLAKE3_KEY_LEN]) {
+  size_t hex_len = strlen(hex_key);
+  if (hex_len != 64) {
+    fprintf(stderr, "Expected a 64-char hexadecimal key, got %zu chars.\n",
+            hex_len);
+    return 1;
+  }
+  for (size_t i = 0; i < 64; i++) {
+    uint8_t value;
+    bool valid;
+    hex_char_value(hex_key[i], &value, &valid);
+    if (!valid) {
+      fprintf(stderr, "Invalid hex char.\n");
+      return 1;
+    }
+    if (i % 2 == 0) {
+      out[i / 2] = 0;
+      value <<= 4;
+    }
+    out[i / 2] += value;
+  }
+  return 0;
+}
+
+/* A little repetition here */
+enum cpu_feature {
+  SSE2 = 1 << 0,
+  SSSE3 = 1 << 1,
+  SSE41 = 1 << 2,
+  AVX = 1 << 3,
+  AVX2 = 1 << 4,
+  AVX512F = 1 << 5,
+  AVX512VL = 1 << 6,
+  /* ... */
+  UNDEFINED = 1 << 30
+};
+
+extern enum cpu_feature g_cpu_features;
+enum cpu_feature get_cpu_features(void);
+
+int main(int argc, char **argv) {
+  size_t out_len = BLAKE3_OUT_LEN;
+  uint8_t key[BLAKE3_KEY_LEN];
+  char *context = "";
+  uint8_t mode = HASH_MODE;
+  while (argc > 1) {
+    if (argc <= 2) {
+      fprintf(stderr, "Odd number of arguments.\n");
+      return 1;
+    }
+    if (strcmp("--length", argv[1]) == 0) {
+      char *endptr = NULL;
+      errno = 0;
+      unsigned long long out_len_ll = strtoull(argv[2], &endptr, 10);
+      if (errno != 0 || out_len_ll > SIZE_MAX || endptr == argv[2] ||
+          *endptr != 0) {
+        fprintf(stderr, "Bad length argument.\n");
+        return 1;
+      }
+      out_len = (size_t)out_len_ll;
+    } else if (strcmp("--keyed", argv[1]) == 0) {
+      mode = KEYED_HASH_MODE;
+      int ret = parse_key(argv[2], key);
+      if (ret != 0) {
+        return ret;
+      }
+    } else if (strcmp("--derive-key", argv[1]) == 0) {
+      mode = DERIVE_KEY_MODE;
+      context = argv[2];
+    } else {
+      fprintf(stderr, "Unknown flag.\n");
+      return 1;
+    }
+    argc -= 2;
+    argv += 2;
+  }
+
+  /*
+   * We're going to hash the input multiple times, so we need to buffer it all.
+   * This is just for test cases, so go ahead and assume that the input is less
+   * than 1 MiB.
+   */
+  size_t buf_capacity = 1 << 20;
+  uint8_t *buf = malloc(buf_capacity);
+  assert(buf != NULL);
+  size_t buf_len = 0;
+  while (1) {
+    size_t n = fread(&buf[buf_len], 1, buf_capacity - buf_len, stdin);
+    if (n == 0) {
+      break;
+    }
+    buf_len += n;
+    assert(buf_len < buf_capacity);
+  }
+
+  const int mask = get_cpu_features();
+  int feature = 0;
+  do {
+    fprintf(stderr, "Testing 0x%08X\n", feature);
+    g_cpu_features = feature;
+    blake3_hasher hasher;
+    switch (mode) {
+    case HASH_MODE:
+      blake3_hasher_init(&hasher);
+      break;
+    case KEYED_HASH_MODE:
+      blake3_hasher_init_keyed(&hasher, key);
+      break;
+    case DERIVE_KEY_MODE:
+      blake3_hasher_init_derive_key(&hasher, context);
+      break;
+    default:
+      abort();
+    }
+
+    blake3_hasher_update(&hasher, buf, buf_len);
+
+    /* TODO: An incremental output reader API to avoid this allocation. */
+    uint8_t *out = malloc(out_len);
+    if (out_len > 0 && out == NULL) {
+      fprintf(stderr, "malloc() failed.\n");
+      return 1;
+    }
+    blake3_hasher_finalize(&hasher, out, out_len);
+    for (size_t i = 0; i < out_len; i++) {
+      printf("%02x", out[i]);
+    }
+    printf("\n");
+    free(out);
+    feature = (feature - mask) & mask;
+  } while (feature != 0);
+  free(buf);
+  return 0;
+}

data/ext/blake3/c/test.py

@@ -0,0 +1,97 @@
+#! /usr/bin/env python3
+
+from binascii import hexlify
+import json
+from os import path
+import subprocess
+
+HERE = path.dirname(__file__)
+TEST_VECTORS_PATH = path.join(HERE, "..", "test_vectors", "test_vectors.json")
+TEST_VECTORS = json.load(open(TEST_VECTORS_PATH))
+
+
+def run_blake3(args, input):
+    output = subprocess.run([path.join(HERE, "blake3")] + args,
+                            input=input,
+                            stdout=subprocess.PIPE,
+                            check=True)
+    return output.stdout.decode().strip()
+
+
+# Fill the input with a repeating byte pattern. We use a cycle length of 251,
+# because that's the largets prime number less than 256. This makes it unlikely
+# to swapping any two adjacent input blocks or chunks will give the same
+# answer.
+def make_test_input(length):
+    i = 0
+    buf = bytearray()
+    while len(buf) < length:
+        buf.append(i)
+        i = (i + 1) % 251
+    return buf
+
+
+def main():
+    for case in TEST_VECTORS["cases"]:
+        input_len = case["input_len"]
+        input = make_test_input(input_len)
+        hex_key = hexlify(TEST_VECTORS["key"].encode())
+        context_string = TEST_VECTORS["context_string"]
+        expected_hash_xof = case["hash"]
+        expected_hash = expected_hash_xof[:64]
+        expected_keyed_hash_xof = case["keyed_hash"]
+        expected_keyed_hash = expected_keyed_hash_xof[:64]
+        expected_derive_key_xof = case["derive_key"]
+        expected_derive_key = expected_derive_key_xof[:64]
+
+        # Test the default hash.
+        test_hash = run_blake3([], input)
+        for line in test_hash.splitlines():
+            assert expected_hash == line, \
+                "hash({}): {} != {}".format(input_len, expected_hash, line)
+
+        # Test the extended hash.
+        xof_len = len(expected_hash_xof) // 2
+        test_hash_xof = run_blake3(["--length", str(xof_len)], input)
+        for line in test_hash_xof.splitlines():
+            assert expected_hash_xof == line, \
+                "hash_xof({}): {} != {}".format(
+                    input_len, expected_hash_xof, line)
+
+        # Test the default keyed hash.
+        test_keyed_hash = run_blake3(["--keyed", hex_key], input)
+        for line in test_keyed_hash.splitlines():
+            assert expected_keyed_hash == line, \
+                "keyed_hash({}): {} != {}".format(
+                    input_len, expected_keyed_hash, line)
+
+        # Test the extended keyed hash.
+        xof_len = len(expected_keyed_hash_xof) // 2
+        test_keyed_hash_xof = run_blake3(
+            ["--keyed", hex_key, "--length",
+             str(xof_len)], input)
+        for line in test_keyed_hash_xof.splitlines():
+            assert expected_keyed_hash_xof == line, \
+                "keyed_hash_xof({}): {} != {}".format(
+                    input_len, expected_keyed_hash_xof, line)
+
+        # Test the default derive key.
+        test_derive_key = run_blake3(["--derive-key", context_string], input)
+        for line in test_derive_key.splitlines():
+            assert expected_derive_key == line, \
+                "derive_key({}): {} != {}".format(
+                    input_len, expected_derive_key, line)
+
+        # Test the extended derive key.
+        xof_len = len(expected_derive_key_xof) // 2
+        test_derive_key_xof = run_blake3(
+            ["--derive-key", context_string, "--length",
+             str(xof_len)], input)
+        for line in test_derive_key_xof.splitlines():
+            assert expected_derive_key_xof == line, \
+                "derive_key_xof({}): {} != {}".format(
+                    input_len, expected_derive_key_xof, line)
+
+
+if __name__ == "__main__":
+    main()

data/lib/uncle_blake3/binding.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+require_relative 'build/loader'
+
+module UncleBlake3
+  module Binding
+    extend ::FFI::Library
+    ffi_lib Build::Loader.find('UncleBlake3')
+
+    attach_function :key_length, :UncleBlake3_KEY_LEN, %i[], :uint16
+    attach_function :default_output_length, :UncleBlake3_OUT_LEN, %i[], :uint16
+
+    attach_function :init, :UncleBlake3_Init, %i[], :pointer
+    attach_function :init_with_key, :UncleBlake3_InitWithKey, %i[pointer], :pointer
+    attach_function :init_with_key_seed, :UncleBlake3_InitWithKeySeed, %i[pointer size_t], :pointer
+    attach_function :update, :UncleBlake3_Update, %i[pointer pointer size_t], :int
+    attach_function :final, :UncleBlake3_Final, %i[pointer pointer size_t], :int
+    attach_function :destroy, :UncleBlake3_Destroy, %i[pointer], :void
+  end
+end

data/lib/uncle_blake3/build/loader.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+require 'ffi'
+require 'pathname'
+require_relative 'platform'
+
+module UncleBlake3
+  module Build
+    # taken from:
+    # https://github.com/ffi/ffi-compiler/blob/master/lib/ffi-compiler/loader.rb
+    module Loader
+      def self.find(name, start_path = nil)
+        library = Platform.instance.map_library_name(name)
+        root = false
+        ::Pathname.new(start_path || caller_path(caller[0])).ascend do |path|
+          Dir.glob("#{path}/**/{#{::FFI::Platform::ARCH}-#{::FFI::Platform::OS}/#{library},#{library}}") do |f|
+            return f
+          end
+
+          break if root
+
+          # Next iteration will be the root of the gem if this is the lib/ dir - stop after that
+          root = ::File.basename(path) == 'lib'
+        end
+        raise ::LoadError, "cannot find '#{name}' library"
+      end
+
+      def self.caller_path(line = caller[0])
+        if ::FFI::Platform::OS == 'windows'
+          drive = line[0..1]
+          path =  line[2..].split(/:/)[0]
+          full_path = drive + path
+        else
+          full_path = line.split(/:/)[0]
+        end
+        ::File.dirname full_path
+      end
+    end
+  end
+end

data/lib/uncle_blake3/build/platform.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require 'ffi'
+require 'singleton'
+
+module UncleBlake3
+  module Build
+    # mostly taken from:
+    # https://github.com/ffi/ffi-compiler/blob/master/lib/ffi-compiler/platform.rb
+
+    class Platform
+      include ::Singleton
+
+      LIBSUFFIX = ::FFI::Platform.mac? ? 'bundle' : ::FFI::Platform::LIBSUFFIX
+
+      def map_library_name(name)
+        "#{::FFI::Platform::LIBPREFIX}#{name}.#{LIBSUFFIX}"
+      end
+
+      def arch
+        ::FFI::Platform::ARCH
+      end
+
+      def os
+        ::FFI::Platform::OS
+      end
+
+      def name
+        ::FFI::Platform.name
+      end
+
+      def mac?
+        ::FFI::Platform.mac?
+      end
+    end
+  end
+end

data/lib/uncle_blake3/build.rb

@@ -0,0 +1,4 @@
+# frozen_string_literal: true
+
+require_relative './build/platform'
+require_relative './build/loader'

data/lib/uncle_blake3/digest.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+require 'base64'
+require 'ffi'
+require 'objspace'
+require_relative 'binding'
+
+module UncleBlake3
+  class Digest
+    KEY_LENGTH = Binding.key_length
+    DEFAULT_OUTPUT_LENGTH = Binding.default_output_length
+
+    module Error
+    end
+
+    class ArgumentError < ::ArgumentError
+      include Error
+    end
+
+    class TypeError < ::TypeError
+      include Error
+    end
+
+    def initialize(output_length: DEFAULT_OUTPUT_LENGTH, key: nil, key_seed: nil)
+      raise TypeError, 'Hash length is not an Integer' unless output_length.is_a?(::Integer)
+      raise ArgumentError, 'Hash length out of range' unless (1...(1 << 20)).include?(output_length)
+      raise TypeError, 'Key is not a String' if !key.nil? && !key.is_a?(::String)
+      raise ArgumentError, 'Key length mismatched' unless key.nil? || (key.bytesize == KEY_LENGTH)
+      raise TypeError, 'Key seed is not a String' if !key_seed.nil? && !key_seed.is_a?(::String)
+      @native_instance = if !key && !key_seed
+        Binding.init
+      elsif key && key_seed
+        raise ::ArgumentError, 'Both key and key_seed available at the same time; please pick only one.'
+      elsif key
+        key_buffer = ::FFI::MemoryPointer.new(:uint8, KEY_LENGTH)
+        key_buffer.put_bytes(0, key)
+        Binding.init_with_key(key_buffer)
+      elsif key_seed
+        seed_size = key_seed.bytesize
+        seed_buffer = ::FFI::MemoryPointer.new(:uint8, seed_size)
+        seed_buffer.put_bytes(0, key_seed)
+        Binding.init_with_key_seed(seed_buffer, seed_size)
+      else
+        raise ::ArgumentError, 'Unknown mode of operation'
+      end.tap do |pointer|
+        ::ObjectSpace.define_finalizer(self, self.class._create_finalizer(pointer))
+      end
+      @output_length = output_length
+      invalidate_cache
+    end
+
+    def update(data)
+      data_size = data.bytesize
+      data_buffer = ::FFI::MemoryPointer.new(:uint8, data_size)
+      data_buffer.put_bytes(0, data)
+      Binding.update(@native_instance, data_buffer, data_size)
+      self
+    ensure
+      invalidate_cache
+    end
+
+    def <<(*args, **kwargs)
+      update(*args, **kwargs)
+    end
+
+    def digest
+      @_digest_cache ||= begin
+        data_buffer = ::FFI::MemoryPointer.new(:uint8, @output_length)
+        Binding.final(@native_instance, data_buffer, @output_length)
+        data_buffer.get_bytes(0, @output_length)
+      end
+    end
+
+    def hexdigest
+      @_hexdigest_cache ||= digest.unpack1('H*')
+    end
+
+    def base64digest
+      @_base64digest_cache ||= ::Base64.strict_encode64(digest)
+    end
+
+    private
+
+    def invalidate_cache
+      @_digest_cache = nil
+      @_hexdigest_cache = nil
+      @_base64digest_cache = nil
+    end
+
+    class << self
+      # https://www.mikeperham.com/2010/02/24/the-trouble-with-ruby-finalizers/
+      def _create_finalizer(instance)
+        proc {
+          Binding.destroy(instance)
+        }
+      end
+
+      def digest(*args, **kwargs)
+        _generic_digest(*args, **kwargs, &:digest)
+      end
+
+      def hexdigest(*args, **kwargs)
+        _generic_digest(*args, **kwargs, &:hexdigest)
+      end
+
+      def base64digest(*args, **kwargs)
+        _generic_digest(*args, **kwargs, &:base64digest)
+      end
+
+      private
+
+      def _generic_digest(data, output_length: nil, key: nil, key_seed: nil, &hash_finalizer)
+        instance = new(**{ output_length: output_length, key: key, key_seed: key_seed }.compact)
+        instance.update(data)
+        hash_finalizer.call(instance)
+      end
+    end
+  end
+end

data/lib/uncle_blake3/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module UncleBlake3
+  VERSION = '0.0.1'
+end

data/lib/uncle_blake3.rb

@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require_relative 'uncle_blake3/version'
+require_relative 'uncle_blake3/digest'
+
+module UncleBlake3
+end

metadata

@@ -0,0 +1,112 @@
+--- !ruby/object:Gem::Specification
+name: uncle_blake3
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Sarun Rattanasiri
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2022-02-17 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ffi
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 1.15.5
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 13.0.6
+description: This gem brought the hash algorithm, Blake3, to Ruby. It uses the official
+  Blake3 implementation which optimized and maintained by the original Blake3 team
+  themselves.
+email: midnight_w@gmx.tw
+executables: []
+extensions:
+- ext/Rakefile
+extra_rdoc_files: []
+files:
+- LICENSE.md
+- README.md
+- ext/Rakefile
+- ext/binding/uncle_blake3.c
+- ext/blake3/c/Makefile.testing
+- ext/blake3/c/README.md
+- ext/blake3/c/blake3.c
+- ext/blake3/c/blake3.h
+- ext/blake3/c/blake3_avx2.c
+- ext/blake3/c/blake3_avx2_x86-64_unix.S
+- ext/blake3/c/blake3_avx2_x86-64_windows_gnu.S
+- ext/blake3/c/blake3_avx2_x86-64_windows_msvc.asm
+- ext/blake3/c/blake3_avx512.c
+- ext/blake3/c/blake3_avx512_x86-64_unix.S
+- ext/blake3/c/blake3_avx512_x86-64_windows_gnu.S
+- ext/blake3/c/blake3_avx512_x86-64_windows_msvc.asm
+- ext/blake3/c/blake3_dispatch.c
+- ext/blake3/c/blake3_impl.h
+- ext/blake3/c/blake3_neon.c
+- ext/blake3/c/blake3_portable.c
+- ext/blake3/c/blake3_sse2.c
+- ext/blake3/c/blake3_sse2_x86-64_unix.S
+- ext/blake3/c/blake3_sse2_x86-64_windows_gnu.S
+- ext/blake3/c/blake3_sse2_x86-64_windows_msvc.asm
+- ext/blake3/c/blake3_sse41.c
+- ext/blake3/c/blake3_sse41_x86-64_unix.S
+- ext/blake3/c/blake3_sse41_x86-64_windows_gnu.S
+- ext/blake3/c/blake3_sse41_x86-64_windows_msvc.asm
+- ext/blake3/c/example.c
+- ext/blake3/c/main.c
+- ext/blake3/c/test.py
+- lib/uncle_blake3.rb
+- lib/uncle_blake3/binding.rb
+- lib/uncle_blake3/build.rb
+- lib/uncle_blake3/build/loader.rb
+- lib/uncle_blake3/build/platform.rb
+- lib/uncle_blake3/digest.rb
+- lib/uncle_blake3/version.rb
+homepage: https://github.com/the-cave/uncle-blake3
+licenses: []
+metadata:
+  homepage_uri: https://github.com/the-cave/uncle-blake3
+  source_code_uri: https://github.com/the-cave/uncle-blake3/tree/v0.0.1
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 2.6.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.2.32
+signing_key:
+specification_version: 4
+summary: A binding of the Blake3 hash algorithm for Ruby
+test_files: []