bloomed 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: c2789813aea4f66b7a6433a8694d60b50d1e7cc5
+  data.tar.gz: be4191d94d992dfddefb9b512461c0a5b6c1f04b
+SHA512:
+  metadata.gz: e8d4bfa81e430496d308a159d8ec513204be4fb2f7939f7e1ff846ee994cae6b7bcf1394b5b6e7a7361e823ef8d0d58f7f92b40818440552f5579400cd90997e
+  data.tar.gz: b136f1cb1b084fb08b3c74e06ee1799de781a7de964f71d3afd2ea91af66d698f8c4f16869c21c368b9a68f66aa58a42de430a438ec3e997b91bb28e71b92c97

data/.circleci/config.yml

@@ -0,0 +1,59 @@
+# Ruby CircleCI 2.0 configuration file
+#
+# Check https://circleci.com/docs/2.0/language-ruby/ for more details
+#
+version: 2
+jobs:
+  build:
+    docker:
+      # specify the version you desire here
+       - image: circleci/ruby:2.5.1-node-browsers
+      
+      # Specify service dependencies here if necessary
+      # CircleCI maintains a library of pre-built images
+      # documented at https://circleci.com/docs/2.0/circleci-images/
+      # - image: circleci/postgres:9.4
+
+    working_directory: ~/repo
+
+    steps:
+      - checkout
+
+      # Download and cache dependencies
+      - restore_cache:
+          keys:
+          - v1-dependencies-{{ checksum "Gemfile.lock" }}
+          # fallback to using the latest cache if no exact match is found
+          - v1-dependencies-
+
+      - run:
+          name: install dependencies
+          command: |
+            bundle install --jobs=4 --retry=3 --path vendor/bundle
+
+      - save_cache:
+          paths:
+            - ./vendor/bundle
+          key: v1-dependencies-{{ checksum "Gemfile.lock" }}
+        
+      # Database setup
+      # - run: bundle exec rake db:create
+      # - run: bundle exec rake db:schema:load
+
+      # run tests!
+      - run:
+          name: run tests
+          command: |
+            mkdir /tmp/test-results
+            TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
+            
+            bundle exec rspec --format progress \
+                            --out /tmp/test-results/rspec.xml \
+                            $TEST_FILES
+
+      # collect reports
+      - store_test_results:
+          path: /tmp/test-results
+      - store_artifacts:
+          path: /tmp/test-results
+          destination: test-results

data/.gitignore

@@ -0,0 +1,11 @@
+/.bundle/
+/.yardoc
+/_yardoc/
+/coverage/
+/doc/
+/pkg/
+/spec/reports/
+/tmp/
+
+# rspec failure tracking
+.rspec_status

data/.rspec

@@ -0,0 +1,3 @@
+--format documentation
+--color
+--require spec_helper

data/.ruby-version

@@ -0,0 +1,2 @@
+
+2.5.1

data/.travis.yml

@@ -0,0 +1,7 @@
+---
+sudo: false
+language: ruby
+cache: bundler
+rvm:
+  - 2.3.7
+before_install: gem install bundler -v 1.16.3

data/CODE_OF_CONDUCT.md

@@ -0,0 +1 @@
+Do as you like

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+
+git_source(:github) {|repo_name| "https://github.com/#{repo_name}" }
+
+# Specify your gem's dependencies in bloomed.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,39 @@
+PATH
+  remote: .
+  specs:
+    bloomed (1.0.0)
+      bloomer
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    bitarray (1.1.0)
+    bloomer (0.0.5)
+      bitarray
+    diff-lcs (1.3)
+    rake (10.5.0)
+    rspec (3.8.0)
+      rspec-core (~> 3.8.0)
+      rspec-expectations (~> 3.8.0)
+      rspec-mocks (~> 3.8.0)
+    rspec-core (3.8.0)
+      rspec-support (~> 3.8.0)
+    rspec-expectations (3.8.1)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-mocks (3.8.0)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.8.0)
+    rspec-support (3.8.0)
+
+PLATFORMS
+  ruby
+
+DEPENDENCIES
+  bloomed!
+  bundler (~> 1.16)
+  rake (~> 10.0)
+  rspec (~> 3.0)
+
+BUNDLED WITH
+   1.16.3

data/LICENSE.txt

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018 Søren Skovsbøll
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

data/README.md

@@ -0,0 +1,89 @@
+# Bloomed
+
+Troy Hunt's brilliant haveibeenpwned.com let's you download SHA1s of 517,238,891 real world passwords previously exposed in data breaches. This list is comprehensive but huge in size: 11GB compressed.
+Using a bloom filter we can reduce the size down to files measured in MBs. 
+
+You can even keep a the bloom filter in memory in your web app or api. This is great if you're afraid to send the passwords that your users enter, to an external service for lookup.
+
+This gem will let you control the trade off between memory size and precision. False positives will occur (that's the nature of bloom filters), but you control the frequency and how many of the pwned passwords you want in your filter, starting from the most pwned at the top.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+```ruby
+gem 'bloomed'
+```
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install bloomed
+
+## Usage
+
+### Quick start
+
+```ruby
+require 'bloomed'
+pw=Bloomed::PW.new
+pw.pwned? "password123"
+=> true
+```
+
+### Using lower precision / lower memory consumption
+
+There are two parameters that can be varied: `top` and `false_positive_probability`.
+
+
+```ruby
+require 'bloomed'
+pw=Bloomed::PW.new(top: 100000, false_positive_probability: 0.01) # 136 kb memory
+pw.pwned? "password123"
+=> true
+```
+
+### Using higher precision / higher memory consumption
+
+To keep the gem size small, it only ships with dumps up to 253 kb in size.
+
+To generate a larger, optimized bloom filter for pwned passwords, please download pwned-passwords-ordered-by-count.7z from https://haveibeenpwned.com and extract `pwned-passwords-ordered-by-count.txt` to the current dir.
+
+Once you have the `pwned-passwords-ordered-by-count.txt` file in place, you can run the following to generate the filter and cache it for later (on your machine.)
+
+```ruby
+require 'bloomed'
+pw=Bloomed::PW.new(top: 1E8, false_positive_probability: 0.0001) # 247 Mb! memory
+pw.pwned? "password123"
+=> true
+```
+
+### The cache
+
+The cache is stored in the `dumps` dir inside `dirname $(gem which bloomed)`.
+
+### Size of the in memory bloom filter
+
+The filter can vary much in size. Use `Bloomed:PW#memory_size_bytes` to get the exact size.
+
+
+## Development
+
+After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
+
+To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
+
+## Contributing
+
+Bug reports and pull requests are welcome on GitHub at https://github.com/skovsboll/bloomed. This project is intended to be a safe, welcoming space for collaboration, and contributors are expected to adhere to the [Contributor Covenant](http://contributor-covenant.org) code of conduct.
+
+## License
+
+The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).
+
+## Code of Conduct
+
+Everyone interacting in the Bloomed project’s codebases, issue trackers, chat rooms and mailing lists is expected to follow the [code of conduct](https://github.com/[USERNAME]/bloomed/blob/master/CODE_OF_CONDUCT.md).

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/console

@@ -0,0 +1,14 @@
+#!/usr/bin/env ruby
+
+require "bundler/setup"
+require "bloomed"
+
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+
+require "irb"
+IRB.start(__FILE__)

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bloomed.gemspec

@@ -0,0 +1,31 @@
+
+lib = File.expand_path("../lib", __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require "bloomed/version"
+
+Gem::Specification.new do |spec|
+  spec.name          = "bloomed"
+  spec.version       = Bloomed::VERSION
+  spec.authors       = ["Søren Skovsbøll"]
+  spec.email         = ["soren@wero.es"]
+
+  spec.summary       = "Bloom filter check for pwned passwords"
+  spec.description   = "Check your users' passwords using a fraction of the memory of the full pwned passwords list."
+  spec.homepage      = "https://github.com/skovsboll/bloomed"
+  spec.license       = "MIT"
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "bloomer"
+
+  spec.add_development_dependency "bundler", "~> 1.16"
+  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rspec", "~> 3.0"
+end

data/lib/bloomed.rb

@@ -0,0 +1,69 @@
+require_relative 'bloomed/version'
+require 'bloomer'
+
+module Bloomed
+  class PW
+    attr_reader :false_positive_probability, :top
+
+    def initialize(top: 100_000, false_positive_probability: 0.001)
+      @top = top
+      @false_positive_probability = false_positive_probability
+      @bloom = prepare
+    end
+
+    def pwned?(pw)
+      @bloom.include?(Digest::SHA1.hexdigest(pw).upcase)
+    end
+
+    def memory_size_bytes
+      require 'objspace'
+      ObjectSpace.memsize_of(@bloom)
+    end
+
+    private
+
+    def prepare
+      if File.exist? file_name
+        load_cache
+      else
+        bloom_filter = generate
+        write_cache bloom_filter
+        bloom_filter
+      end
+    end
+
+    def file_name
+      inverse_probaility = 1.0 / false_positive_probability
+      File.join(
+        File.dirname(__FILE__),
+        'dump',
+        "pwned_top_#{top.to_i}_one_in_#{inverse_probaility.to_i}.dump"
+      )
+    end
+
+    def load_cache
+      Marshal.load(File.read(file_name))
+    end
+
+    PWNED_PASSWORDS_FILENAME = 'pwned-passwords-ordered-by-count.txt'.freeze
+    def generate
+      unless File.exist? PWNED_PASSWORDS_FILENAME
+        raise "In order to generate new, optimized bloom filter for
+        pwned passwords, please download pwned-passwords-ordered-by-count.7z from
+        https://haveibeenpwned.com and extract pwned-passwords-ordered-by-count.txt
+        to the current dir."
+      end
+
+      bloom_filter = Bloomer::Scalable.new(top, false_positive_probability)
+      File.open(PWNED_PASSWORDS_FILENAME, 'r').first(top).each do |line|
+        bloom_filter.add line[0...40]
+      end
+      bloom_filter
+    end
+
+    def write_cache(b)
+      File.write(file_name, Marshal.dump(b))
+      b
+    end
+  end
+end

data/lib/bloomed/version.rb

@@ -0,0 +1,3 @@
+module Bloomed
+  VERSION = "1.0.0"
+end

data/lib/seed.rb

@@ -0,0 +1,8 @@
+require './lib/bloomed'
+
+ps = [0.01, 0.001, 0.0001]
+cs = [1E4, 1E5, 1E6, 1E7, 1E8]
+
+cs.product(ps).each do |c, p|
+  b = Bloomed::PW.new(first: c, false_positive_probability: p)
+end

metadata

@@ -0,0 +1,124 @@
+--- !ruby/object:Gem::Specification
+name: bloomed
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Søren Skovsbøll
+autorequire: 
+bindir: exe
+cert_chain: []
+date: 2018-09-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: bloomer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.16'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '10.0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+description: Check your users' passwords using a fraction of the memory of the full
+  pwned passwords list.
+email:
+- soren@wero.es
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".circleci/config.yml"
+- ".gitignore"
+- ".rspec"
+- ".ruby-version"
+- ".travis.yml"
+- CODE_OF_CONDUCT.md
+- Gemfile
+- Gemfile.lock
+- LICENSE.txt
+- README.md
+- Rakefile
+- bin/console
+- bin/setup
+- bloomed.gemspec
+- lib/bloomed.rb
+- lib/bloomed/version.rb
+- lib/dump/pwned_top_100000_one_in_100.dump
+- lib/dump/pwned_top_100000_one_in_1000.dump
+- lib/dump/pwned_top_100000_one_in_10000.dump
+- lib/dump/pwned_top_10000_one_in_100.dump
+- lib/dump/pwned_top_10000_one_in_1000.dump
+- lib/dump/pwned_top_10000_one_in_10000.dump
+- lib/seed.rb
+homepage: https://github.com/skovsboll/bloomed
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.5.2.3
+signing_key: 
+specification_version: 4
+summary: Bloom filter check for pwned passwords
+test_files: []