pwnedkeys-filter 0.1.0

data/README.md

@@ -0,0 +1,110 @@
+This is a gem for creating and querying [pwnedkeys.com](https://pwnedkeys.com)
+[bloom filters](https://pwnedkeys.com/filter.html).
+
+
+# Installation
+
+It's a gem:
+
+    gem install pwnedkeys-filter
+
+There's also the wonders of [the Gemfile](http://bundler.io):
+
+    gem 'pwnedkeys-filter'
+
+If you're the sturdy type that likes to run from git:
+
+    rake install
+
+Or, if you've eschewed the convenience of Rubygems entirely, then you
+presumably know what to do already.
+
+
+# Usage
+
+Before you do anything, load the code:
+
+    require "pwnedkeys/filter"
+
+The following illustrative examples do not show the full capabilities of the
+gem.  The [API documentation](https://rubydoc.info/gems/pwnedkeys-filter) gives
+all the possibilities.
+
+
+## Querying the filter
+
+Open the filter file, then call `#probably_includes?` with a key:
+
+    filter = Pwnedkeys::Filter.open("/path/to/the/filter/data/file")
+    key    = OpenSSL::PKey::RSA.new(2048)
+
+    # if this returns `true`, it's not your lucky day
+    filter.probably_includes?(key)
+
+    # From https://pwnedkeys.com/examples/rsa2048_key.pem
+    key = OpenSSL::PKey.read(File.read("pwnedkeys_demo_rsa2048.pem"))
+
+    # This should definitely return `true`
+    filter.probably_includes?(key)
+
+Essentially, when passed anything that looks suspiciously like a key, the
+`#probably_includes?` method will return `true` if the key is *probably* in the
+dataset, and will return `false` if the key is *definitely not* in the dataset.
+For more on why the answers are "definitely not" and "probably", it's best to
+read [the wikipedia page on bloom
+filters](https://en.wikipedia.org/wiki/Bloom_filter).
+
+
+## Creating a new filter file
+
+To create a new filter data file:
+
+    Pwnedkeys::Filter.create("/some/file/name", hash_count: 4, hash_length: 12)
+
+This does not return an open filter file, it merely creates a new file on
+disk, initialized appropriately.
+
+
+## Adding new entries to a filter file
+
+To add entries to an open filter:
+
+    filter.add(key)
+
+When you're done adding entries:
+
+    filter.close
+
+The API takes care of updating the filter metadata automatically.
+
+If you don't want to have to manually close the filter (and who does, really?),
+then the block form of `#open` is for you:
+
+    Pwnedkeys::Filter.open("/filter/file") do |filter|
+      filter.add(key)
+    end
+
+
+# Contributing
+
+See [`CONTRIBUTING.md`](CONTRIBUTING.md).
+
+
+# Licence
+
+Unless otherwise stated, everything in this repo is covered by the following
+copyright notice:
+
+    Copyright (C) 2019  Matt Palmer <matt@hezmatt.org>
+
+    This program is free software: you can redistribute it and/or modify it
+    under the terms of the GNU General Public License version 3, as
+    published by the Free Software Foundation.
+
+    This program is distributed in the hope that it will be useful,
+    but WITHOUT ANY WARRANTY; without even the implied warranty of
+    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+    GNU General Public License for more details.
+
+    You should have received a copy of the GNU General Public License
+    along with this program.  If not, see <http://www.gnu.org/licenses/>.

data/lib/pwnedkeys/filter.rb

@@ -0,0 +1,309 @@
+require "xxhash"
+require "openssl/x509/spki"
+
+module Pwnedkeys
+  class Filter
+
+    # Base class for all Pwnedkeys::Filter exceptions
+    class Error < StandardError; end
+
+    # Raised when a data file appears to not be a valid data file.
+    class InvalidFileError < Error; end
+
+    # Raised when the key to be searched for isn't a key
+    class InvalidKeyError < Error; end
+
+    # Attempt was made to query or modify a closed filter
+    class FilterClosedError < Error; end
+
+    class Header
+      attr_reader :signature, :revision, :update_time, :entry_count, :hash_count, :hash_length
+
+      def self.from_fd(fd)
+        fd.seek(0)
+        case fd.read(6)
+        when "pkbfv1"
+          V1Header.from_fd(fd)
+        else
+          raise InvalidFileError,
+                "No recognised file signature found"
+        end
+      end
+
+      def initialize(**params)
+        @revision    = 0
+        @update_time = Time.at(0)
+        @entry_count = 0
+
+        params.each do |k, v|
+          instance_variable_set(:"@#{k}", v)
+        end
+      end
+
+      def header_size
+        to_s.length
+      end
+
+      def update!
+        @revision += 1
+      end
+
+      def entry_added!
+        @update_time = Time.now
+        @entry_count += 1
+      end
+    end
+    private_constant :Header
+
+    class V1Header < Header
+      def self.from_fd(fd)
+        fd.seek(0)
+        signature, revision, update, entry_count, hash_count, hash_length = fd.read(24).unpack("a6L>Q>L>CC")
+        update_time = Time.at(update)
+
+        self.new(
+          signature:   signature,
+          revision:    revision,
+          update_time: update_time,
+          entry_count: entry_count,
+          hash_count:  hash_count,
+          hash_length: hash_length
+        )
+      end
+
+      def initialize(**params)
+        @signature = "pkbfv1"
+
+        super
+      end
+
+      def to_s
+        [@signature, @revision, @update_time.to_i, @entry_count, @hash_count, @hash_length].pack("a*L>Q>L>CC")
+      end
+    end
+    private_constant :V1Header
+
+    # Create a new filter data file.
+    #
+    # Initialize a file, which cannot already exist, to be a pwnedkeys bloom filter
+    # v1 file.  The file is not opened for use, it is simply created on the filesystem
+    # at the location specified.
+    #
+    # @param filename [String] the file to be created.  It can be an absolute or relative
+    #    path, or anything else that `File.open` will accept.
+    #
+    # @param hash_count [Integer] how many filter bits each element in the bloom filter will
+    #    set.
+    #
+    # @param hash_length [Integer] the number of bits that will be used for each hash value.
+    #
+    # @raise [SystemCallError] if any sort of filesystem-related problem occurs, an
+    #    `Errno`-related exception will be raised.  Likely candidates include `EEXIST`
+    #    (the file you specified already exists), `ENOENT` (the directory you specified
+    #    doesn't exist), and `EPERM` (you don't have permissions to create a file where
+    #    you want it).
+    #
+    # @return [void]
+    #
+    def self.create(filename, hash_count:, hash_length:)
+      File.open(filename, File::WRONLY | File::CREAT | File::EXCL) do |fd|
+        header = V1Header.new(hash_count: hash_count, hash_length: hash_length)
+
+        fd.write(header.to_s)
+        fd.seek((2 ** hash_length) / 8 - 1, :CUR)
+        fd.write("\0")
+      end
+
+      nil
+    end
+
+    # Open an existing pwnedkeys bloom filter data file.
+    #
+    # @param filename [String] the file to open.
+    #
+    # @raise [SystemCallError] if anything low-level goes wrong, you will get some
+    #    sort of `Errno`-related exception raised, such as `ENOENT` (the file you
+    #    specified does not exist) or `EPERM` (you don't have access to the file
+    #    specified).
+    #
+    # @raise [Pwnedkeys::Filter::InvalidFileError] if the specified file exists,
+    #    but is not recognised as a valid pwnedkeys filter file.
+    #
+    # @return [Pwnedkeys::Filter]
+    #
+    def self.open(filename)
+      filter = Pwnedkeys::Filter.new(filename)
+
+      if block_given?
+        yield filter
+      else
+        filter
+      end
+    end
+
+    # Create a new Pwnedkeys::Filter.
+    #
+    # Equivalent to {.open}, without the possibility of block-style access.
+    #
+    # @see .open
+    #
+    def initialize(filename)
+      @fd     = File.open(filename, File::RDWR, binmode: true)
+      @header = Header.from_fd(@fd)
+    end
+
+    # Query the bloom filter.
+    #
+    # @param key [OpenSSL::PKey::PKey, OpenSSL::X509::SPKI, String] the key
+    #    to query the filter for.
+    #
+    # @return [Boolean] whether the queried key *probably* exists in the
+    #    filter (`true`), or whether it *definitely doesn't* (`false`).
+    #
+    # @raise [Pwnedkeys::Filter::InvalidKeyError] if the object passed in
+    #    isn't recognised as a key.
+    #
+    # @raise [Pwnedkeys::Filter::FilterClosedError] if you try to query
+    #    a filter object which has had {#close} called on it.
+    #
+    def probably_includes?(key)
+      raise FilterClosedError if @fd.nil?
+
+      spki = spkify(key)
+      filter_bits(spki.to_der).all?
+    end
+
+    # Add a new key (or SPKI) to the filter.
+    #
+    # @param key [OpenSSL::PKey::PKey, OpenSSL::X509::SPKI, String] the key
+    #    to add to the filter.
+    #
+    # @return [Boolean] whether the key was added as a new entry.  Due to the
+    #    probabilistic nature of the bloom filter structure, it is possible to
+    #    add two completely different keys and yet it looks like the "same"
+    #    key to the bloom filter.  Adding two colliding keys isn't a fatal
+    #    error, but it is a hint that perhaps the existing filter is getting
+    #    a little too full.
+    #
+    # @raise [Pwnedkeys::Filter::FilterClosedError] if you try to add a key
+    #    to a filter object which has had {#close} called on it.
+    #
+    def add(key)
+      raise FilterClosedError if @fd.nil?
+
+      return false if probably_includes?(key)
+
+      spki = spkify(key)
+
+      filter_positions(spki.to_der).each do |n|
+        @fd.seek(n / 8 + @header.header_size, :SET)
+        byte = @fd.read(1).ord
+        @fd.seek(-1, :CUR)
+
+        mask = 2 ** (7 - (n % 8))
+        new_byte = byte | mask
+
+        @fd.write(new_byte.chr)
+      end
+
+      @header.entry_added!
+
+      # Only update the revision if this is the first add in this filter,
+      # because otherwise the revision counter would just be the same as the
+      # entry counter, and that would be pointless.
+      unless @already_modified
+        @header.update!
+      end
+
+      @fd.seek(0)
+      @fd.write(@header.to_s)
+
+      @already_modified = true
+    end
+
+    # Signal that the filter should be closed for further querying and manipulation.
+    #
+    # @return [void]
+    #
+    # @raise [SystemCallError] if something filesystem-ish fails.
+    #
+    def close
+      @fd.close
+      @fd = nil
+    end
+
+    # An estimate of the false-positive rate inherent in the filter.
+    #
+    # Given the parameters of the filter, we can estimate roughly what the
+    # false-positive rate will be when querying this filter.
+    #
+    # @return [Float] the approximate probability of a query result being a
+    #    false positive, expressed as a floating-point number between 0 and 1.
+    #
+    def false_positive_rate
+      # Taken wholesale from https://en.wikipedia.org/wiki/Bloom_filter#Probability_of_false_positives
+      (1 - (1 - 1.0 / filter_bit_count) ** (hash_count * entry_count)) ** hash_count
+    end
+
+
+    private
+
+    def hash_count
+      @header.hash_count
+    end
+
+    def hash_length
+      @header.hash_length
+    end
+
+    def entry_count
+      @header.entry_count
+    end
+
+    def filter_bit_count
+      @filter_size ||= (2 ** hash_length)
+    end
+
+    def spkify(key)
+      if key.is_a?(OpenSSL::X509::SPKI)
+        key
+      elsif key.is_a?(OpenSSL::PKey::PKey)
+        key.to_spki
+      elsif key.is_a?(String)
+        begin
+          OpenSSL::PKey.read(key).to_spki
+        rescue OpenSSL::ASN1::ASN1Error, OpenSSL::PKey::PKeyError
+          begin
+            OpenSSL::X509::SPKI.new(key)
+          rescue OpenSSL::ASN1::ASN1Error, OpenSSL::X509::SPKIError
+            raise InvalidKeyError,
+                  "Could not parse provided key as a key or SPKI structure"
+          end
+        end
+      else
+        raise InvalidKeyError,
+              "Did not recognise the provided key"
+      end
+    end
+
+    def filter_bits(s)
+      filter_positions(s).map do |n|
+        @fd.seek(n / 8 + @header.header_size, :SET)
+        byte = @fd.read(1).ord
+        mask = 2 ** (7 - (n % 8))
+
+        (byte & mask) > 0
+      end
+    end
+
+    def filter_positions(s)
+      h1 = XXhash.xxh64(s, 0)
+      h2 = XXhash.xxh64(s, 1)
+      h2 += 1 if h2 % 2 == 0
+
+      (0..hash_count-1).map do |i|
+        (h1 + i * h2 + (i ** 3 - i) / 6) % filter_bit_count
+      end
+    end
+  end
+end

data/pwnedkeys-filter.gemspec

@@ -0,0 +1,38 @@
+begin
+  require 'git-version-bump'
+rescue LoadError
+  nil
+end
+
+Gem::Specification.new do |s|
+  s.name = "pwnedkeys-filter"
+
+  s.version = GVB.version rescue "0.0.0.1.NOGVB"
+  s.date    = GVB.date    rescue Time.now.strftime("%Y-%m-%d")
+
+  s.platform = Gem::Platform::RUBY
+
+  s.summary  = "Library to query pwnedkeys.com bloom filters"
+
+  s.authors  = ["Matt Palmer"]
+  s.email    = ["matt@hezmatt.org"]
+  s.homepage = "https://github.com/pwnedkeys/pwnedkeys-filter"
+
+  s.files = `git ls-files -z`.split("\0").reject { |f| f =~ /^(\.|G|spec|Rakefile)/ }
+
+  s.required_ruby_version = ">= 2.5.0"
+
+  s.add_runtime_dependency "openssl-additions"
+  s.add_runtime_dependency "xxhash"
+
+  s.add_development_dependency 'bundler'
+  s.add_development_dependency 'github-release'
+  s.add_development_dependency 'git-version-bump'
+  s.add_development_dependency 'guard-rspec'
+  s.add_development_dependency 'rack-test'
+  s.add_development_dependency 'rake', "~> 12.0"
+  s.add_development_dependency 'redcarpet'
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'simplecov'
+  s.add_development_dependency 'yard'
+end