bloomed 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: c2789813aea4f66b7a6433a8694d60b50d1e7cc5
-  data.tar.gz: be4191d94d992dfddefb9b512461c0a5b6c1f04b
+SHA256:
+  metadata.gz: 527059343640b966e5fb66bd63bbee69553efc8179e6dc5dbcf488461358c010
+  data.tar.gz: 35e947c41fab4771cbf58654b806cd984c054ef59764f46f4a44a9440d1f5d49
 SHA512:
-  metadata.gz: e8d4bfa81e430496d308a159d8ec513204be4fb2f7939f7e1ff846ee994cae6b7bcf1394b5b6e7a7361e823ef8d0d58f7f92b40818440552f5579400cd90997e
-  data.tar.gz: b136f1cb1b084fb08b3c74e06ee1799de781a7de964f71d3afd2ea91af66d698f8c4f16869c21c368b9a68f66aa58a42de430a438ec3e997b91bb28e71b92c97
+  metadata.gz: e3aecc9a6492e27649e18d79dd3331df0f1bc93b8e375c5307474d4786056eac58684f5645407ba1c4826c87909029093603f7b37fdba229de216558af602782
+  data.tar.gz: 2633bc230b66f0d8240dbe75e4a3d0519690db6919383b03b7881524c3f1a58fea25f4e1b03931df6f0d0f2061b7c43986b1d51c0bc823a84e67749282baacb6

data/.circleci/config.yml

@@ -8,7 +8,7 @@ jobs:
     docker:
       # specify the version you desire here
        - image: circleci/ruby:2.5.1-node-browsers
-      
+
       # Specify service dependencies here if necessary
       # CircleCI maintains a library of pre-built images
       # documented at https://circleci.com/docs/2.0/circleci-images/
@@ -35,7 +35,7 @@ jobs:
           paths:
             - ./vendor/bundle
           key: v1-dependencies-{{ checksum "Gemfile.lock" }}
-        
+
       # Database setup
       # - run: bundle exec rake db:create
       # - run: bundle exec rake db:schema:load
@@ -44,16 +44,14 @@ jobs:
       - run:
           name: run tests
           command: |
-            mkdir /tmp/test-results
             TEST_FILES="$(circleci tests glob "spec/**/*_spec.rb" | circleci tests split --split-by=timings)"
-            
-            bundle exec rspec --format progress \
-                            --out /tmp/test-results/rspec.xml \
-                            $TEST_FILES
+            mkdir -p /tmp/test_results/rspec
+            bundle exec rspec --profile 10 \
+                              --format RspecJunitFormatter \
+                              --out /tmp/test_results/rspec/results.xml \
+                              --format progress \
+                              $TEST_FILES
 
       # collect reports
       - store_test_results:
-          path: /tmp/test-results
-      - store_artifacts:
-          path: /tmp/test-results
-          destination: test-results
+          path: /tmp/test_results

data/Gemfile.lock

@@ -2,16 +2,52 @@ PATH
   remote: .
   specs:
     bloomed (1.0.0)
-      bloomer
+      bloomer (~> 1.0)
+      msgpack
 
 GEM
   remote: https://rubygems.org/
   specs:
-    bitarray (1.1.0)
-    bloomer (0.0.5)
+    bitarray (1.2.0)
+    bloomer (1.0.0)
       bitarray
+      msgpack
+    coderay (1.1.2)
     diff-lcs (1.3)
-    rake (10.5.0)
+    ffi (1.9.25)
+    formatador (0.2.5)
+    guard (2.14.2)
+      formatador (>= 0.2.4)
+      listen (>= 2.7, < 4.0)
+      lumberjack (>= 1.0.12, < 2.0)
+      nenv (~> 0.1)
+      notiffany (~> 0.0)
+      pry (>= 0.9.12)
+      shellany (~> 0.0)
+      thor (>= 0.18.1)
+    guard-compat (1.2.1)
+    guard-rspec (4.7.3)
+      guard (~> 2.1)
+      guard-compat (~> 1.1)
+      rspec (>= 2.99.0, < 4.0)
+    listen (3.1.5)
+      rb-fsevent (~> 0.9, >= 0.9.4)
+      rb-inotify (~> 0.9, >= 0.9.7)
+      ruby_dep (~> 1.2)
+    lumberjack (1.0.13)
+    method_source (0.9.0)
+    msgpack (1.2.4)
+    nenv (0.3.0)
+    notiffany (0.1.1)
+      nenv (~> 0.1)
+      shellany (~> 0.0)
+    pry (0.11.3)
+      coderay (~> 1.1.0)
+      method_source (~> 0.9.0)
+    rake (12.3.1)
+    rb-fsevent (0.10.3)
+    rb-inotify (0.9.10)
+      ffi (>= 0.5.0, < 2)
     rspec (3.8.0)
       rspec-core (~> 3.8.0)
       rspec-expectations (~> 3.8.0)
@@ -25,6 +61,11 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
+    rspec_junit_formatter (0.4.1)
+      rspec-core (>= 2, < 4, != 2.12.0)
+    ruby_dep (1.5.0)
+    shellany (0.0.1)
+    thor (0.20.0)
 
 PLATFORMS
   ruby
@@ -32,8 +73,11 @@ PLATFORMS
 DEPENDENCIES
   bloomed!
   bundler (~> 1.16)
-  rake (~> 10.0)
+  guard
+  guard-rspec
+  rake (~> 12.0)
   rspec (~> 3.0)
+  rspec_junit_formatter
 
 BUNDLED WITH
-   1.16.3
+   1.16.4

data/README.md

@@ -1,7 +1,7 @@
 # Bloomed
 
 Troy Hunt's brilliant haveibeenpwned.com let's you download SHA1s of 517,238,891 real world passwords previously exposed in data breaches. This list is comprehensive but huge in size: 11GB compressed.
-Using a bloom filter we can reduce the size down to files measured in MBs. 
+Using a bloom filter we can reduce the size down to files measured in MBs.
 
 You can even keep a the bloom filter in memory in your web app or api. This is great if you're afraid to send the passwords that your users enter, to an external service for lookup.
 
@@ -29,7 +29,7 @@ Or install it yourself as:
 
 ```ruby
 require 'bloomed'
-pw=Bloomed::PW.new
+pw = Bloomed::PW.new
 pw.pwned? "password123"
 => true
 ```
@@ -41,7 +41,7 @@ There are two parameters that can be varied: `top` and `false_positive_probabili
 
 ```ruby
 require 'bloomed'
-pw=Bloomed::PW.new(top: 100000, false_positive_probability: 0.01) # 136 kb memory
+pw = Bloomed::PW.new(top: 100000, false_positive_probability: 0.01) # 136 kb memory
 pw.pwned? "password123"
 => true
 ```
@@ -50,20 +50,38 @@ pw.pwned? "password123"
 
 To keep the gem size small, it only ships with dumps up to 253 kb in size.
 
-To generate a larger, optimized bloom filter for pwned passwords, please download pwned-passwords-ordered-by-count.7z from https://haveibeenpwned.com and extract `pwned-passwords-ordered-by-count.txt` to the current dir.
+To generate all combinations of `top` and `false_positive_probability` bloom filters for pwned passwords, run:
 
-Once you have the `pwned-passwords-ordered-by-count.txt` file in place, you can run the following to generate the filter and cache it for later (on your machine.)
+`rake seed\[all\]`
+
+This will download the source 7zip file with pwned passwords, unpack it to the current dir, write the generated bloom filters in the lib/dump dir relative to the installation path of the gem.
+
+Note: You'll need to `brew install curl p7zip` on macos and `apt-get install curl p7zip` on linux.
+
+Sometimes you will want to have more precise control of the placement of the cache files. To seed all variants in the current dir, run:
+
+```
+rake seed_here\[all\]
+```
+
+But be aware that it will take a long time!
+
+Once you have the massive 22GB text files available, you can generate binary cache files using the exact precision you want.
 
 ```ruby
 require 'bloomed'
-pw=Bloomed::PW.new(top: 1E8, false_positive_probability: 0.0001) # 247 Mb! memory
-pw.pwned? "password123"
-=> true
+pw = Bloomed::PW.new(top:1E9, false_positive_probability: 0.0001)
 ```
 
-### The cache
+Warning! This seeds all the passwords and will take a loooong time the first time. Even with a binary cache file in place loading it will take massive time and memory.
+
+For deployment scenarios where you don't want the server to `rake seed`, you can override the directory used for caching by giving a `cache_dir` argument to the constructor:
 
-The cache is stored in the `dumps` dir inside `dirname $(gem which bloomed)`.
+```ruby
+require 'bloomed'
+pw = Bloomed::PW.new(top:1E8, false_positive_probability: 0.0001, cache_dir: '/var/lib/bloomed'
+)
+```
 
 ### Size of the in memory bloom filter
 

data/Rakefile

@@ -1,6 +1,21 @@
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
+require_relative 'lib/seeder'
 
 RSpec::Core::RakeTask.new(:spec)
 
 task :default => :spec
+
+task :download do
+  unless File.exist? "pwned-passwords-ordered-by-count.txt"
+    `curl -O https://downloads.pwnedpasswords.com/passwords/pwned-passwords-ordered-by-count.7z; 7za e pwned-passwords-ordered-by-count.7z; rm pwned-passwords-ordered-by-count.7z;`
+  end
+end
+
+task :seed, [:all] => [:download] do |_, args|
+  Seeder.seed all: args[:all]
+end
+
+task :seed_here, [:all] => [:download] do |_, args|
+  Seeder.seed all: args[:all], cache_dir: Dir.getwd
+end

data/bloomed.gemspec

@@ -23,9 +23,13 @@ Gem::Specification.new do |spec|
   spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]
 
-  spec.add_dependency "bloomer"
+  spec.add_dependency "bloomer", "~> 1.0"
+  spec.add_dependency "msgpack"
 
   spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
+  spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
+  spec.add_development_dependency "guard"
+  spec.add_development_dependency "guard-rspec"
+  spec.add_development_dependency "rspec_junit_formatter"
 end

data/lib/bloomed.rb

@@ -1,11 +1,22 @@
-require_relative 'bloomed/version'
-require 'bloomer'
+require_relative "bloomed/version"
+require_relative "bloomed/msg_packable"
+require "bloomer"
+require "bloomer/msgpackable"
 
 module Bloomed
   class PW
-    attr_reader :false_positive_probability, :top
+    include MsgPackable::Bloomed
 
-    def initialize(top: 100_000, false_positive_probability: 0.001)
+    attr_reader :false_positive_probability,
+                :top,
+                :bloom,
+                :cache_dir,
+                :filename
+
+    def initialize(top: 100_000,
+                   false_positive_probability: 0.001,
+                   cache_dir: nil)
+      @cache_dir = cache_dir || File.join(File.dirname(__FILE__), "dump")
       @top = top
       @false_positive_probability = false_positive_probability
       @bloom = prepare
@@ -16,14 +27,20 @@ module Bloomed
     end
 
     def memory_size_bytes
-      require 'objspace'
+      require "objspace"
       ObjectSpace.memsize_of(@bloom)
     end
 
+    def filename
+      inverse_probaility = 1.0 / false_positive_probability
+      File.join(cache_dir,
+                "pwned_top_#{top.to_i}_one_in_#{inverse_probaility.to_i}.msgpk")
+    end
+
     private
 
     def prepare
-      if File.exist? file_name
+      if File.exist? filename
         load_cache
       else
         bloom_filter = generate
@@ -32,38 +49,30 @@ module Bloomed
       end
     end
 
-    def file_name
-      inverse_probaility = 1.0 / false_positive_probability
-      File.join(
-        File.dirname(__FILE__),
-        'dump',
-        "pwned_top_#{top.to_i}_one_in_#{inverse_probaility.to_i}.dump"
-      )
-    end
-
     def load_cache
-      Marshal.load(File.read(file_name))
+      Bloomed::PW.from_msgpack(File.read(filename))
     end
 
-    PWNED_PASSWORDS_FILENAME = 'pwned-passwords-ordered-by-count.txt'.freeze
+    PWNED_PASSWORDS_FILENAME = "pwned-passwords-ordered-by-count.txt".freeze
+
     def generate
       unless File.exist? PWNED_PASSWORDS_FILENAME
-        raise "In order to generate new, optimized bloom filter for
-        pwned passwords, please download pwned-passwords-ordered-by-count.7z from
-        https://haveibeenpwned.com and extract pwned-passwords-ordered-by-count.txt
-        to the current dir."
+        raise MissingPasswordListError, "To generate new, optimized pwned passwords bloom filter, run `rake seed` or `rake seed_here`. To generate all variants, even the very large binaries, run `rake seed\[all\]` or `rake seed_here\[all\]`."
       end
 
-      bloom_filter = Bloomer::Scalable.new(top, false_positive_probability)
-      File.open(PWNED_PASSWORDS_FILENAME, 'r').first(top).each do |line|
+      bloom_filter = Bloomer.new(top, false_positive_probability)
+      File.open(PWNED_PASSWORDS_FILENAME, "r").first(top).each do |line|
         bloom_filter.add line[0...40]
       end
       bloom_filter
     end
 
     def write_cache(b)
-      File.write(file_name, Marshal.dump(b))
+      File.write(filename, b.to_msgpack)
       b
     end
   end
+
+  class MissingPasswordListError < RuntimeError
+  end
 end

data/lib/bloomed/msg_packable.rb

@@ -0,0 +1,45 @@
+require 'msgpack'
+
+module MsgPackable
+  module Bloomed
+    def self.included(base)
+      base.extend(ClassMethods)
+    end
+
+    def to_msgpack_ext
+      self.class.msgpack_factory.dump([@top, @false_positive_probability, @bloom])
+    end
+
+    def from_msgpack_ext(top, false_positive_probability, bloom)
+      @top = top
+      @false_positive_probability = false_positive_probability
+      @bloom = bloom
+    end
+
+    def to_msgpack
+      self.class.msgpack_factory.dump self
+    end
+
+    module ClassMethods
+      def from_msgpack(data)
+        msgpack_factory.load(data)
+      end
+
+      def from_msgpack_ext(data)
+        values = msgpack_factory.load(data)
+        ::Bloomed::PW.new.tap do |b|
+          b.from_msgpack_ext(*values)
+        end
+      end
+
+      def msgpack_factory
+        @msgpack_factory ||= ::MessagePack::Factory.new.tap do |factory|
+          factory.register_type(0x01, ::Bloomer)
+          factory.register_type(0x02, ::Bloomer::Scalable)
+          factory.register_type(0x03, ::Bloomed::PW)
+          factory.freeze
+        end
+      end
+    end
+  end
+end

data/lib/bloomed/version.rb

@@ -1,3 +1,3 @@
 module Bloomed
-  VERSION = "1.0.0"
+  VERSION = "1.1.0"
 end

data/lib/seeder.rb

@@ -0,0 +1,21 @@
+require "./lib/bloomed"
+
+KB = 2 ** 10
+
+module Seeder
+  def self.seed(all: false, cache_dir: nil)
+    ps = [0.01, 0.001, 0.0001]
+    cs = [1E4, 1E5]
+
+    if all
+      cs += [1E6, 1E7, 1E8]
+    end
+
+    cs.product(ps).each do |c, p|
+      print "Generating filter for top #{c.to_i} with #{p} precision "
+      b = Bloomed::PW.new(top: c, false_positive_probability: p, cache_dir: cache_dir)
+      size = File.size(b.filename)
+      puts "Done [#{size / KB} kb]"
+    end
+  end
+end

metadata

@@ -1,17 +1,31 @@
 --- !ruby/object:Gem::Specification
 name: bloomed
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Søren Skovsbøll
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-09-02 00:00:00.000000000 Z
+date: 2018-09-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bloomer
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -44,14 +58,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '12.0'
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -66,6 +80,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: guard
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: guard-rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec_junit_formatter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description: Check your users' passwords using a fraction of the memory of the full
   pwned passwords list.
 email:
@@ -89,14 +145,15 @@ files:
 - bin/setup
 - bloomed.gemspec
 - lib/bloomed.rb
+- lib/bloomed/msg_packable.rb
 - lib/bloomed/version.rb
-- lib/dump/pwned_top_100000_one_in_100.dump
-- lib/dump/pwned_top_100000_one_in_1000.dump
-- lib/dump/pwned_top_100000_one_in_10000.dump
-- lib/dump/pwned_top_10000_one_in_100.dump
-- lib/dump/pwned_top_10000_one_in_1000.dump
-- lib/dump/pwned_top_10000_one_in_10000.dump
-- lib/seed.rb
+- lib/dump/pwned_top_100000_one_in_100.msgpk
+- lib/dump/pwned_top_100000_one_in_1000.msgpk
+- lib/dump/pwned_top_100000_one_in_10000.msgpk
+- lib/dump/pwned_top_10000_one_in_100.msgpk
+- lib/dump/pwned_top_10000_one_in_1000.msgpk
+- lib/dump/pwned_top_10000_one_in_10000.msgpk
+- lib/seeder.rb
 homepage: https://github.com/skovsboll/bloomed
 licenses:
 - MIT
@@ -117,7 +174,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.5.2.3
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
 summary: Bloom filter check for pwned passwords

data/lib/seed.rb

@@ -1,8 +0,0 @@
-require './lib/bloomed'
-
-ps = [0.01, 0.001, 0.0001]
-cs = [1E4, 1E5, 1E6, 1E7, 1E8]
-
-cs.product(ps).each do |c, p|
-  b = Bloomed::PW.new(first: c, false_positive_probability: p)
-end