rangefinder 0.0.1

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    ODRiODljY2Q5MmFhZjBjYzg1ZjI5Y2YxMDgyZmY1YWZiOTk0MTA4YQ==
+  data.tar.gz: !binary |-
+    YTgwODAzMjA3ODNjNTZmYjZkYWNhMGQ3YjM4MTdjYmMzNzFmN2RmMg==
+SHA512:
+  metadata.gz: !binary |-
+    NDk5NWUwNWI0NGMyMzRlMTcwYWI5OGQ0N2M5NGNiMzU2MTc5ZDM1NmFkMTU2
+    NWRmZTRiYWMwOTVlZmFjZTc2OGQyZDY5ODdjMzk2ODk0Yjg5MjAyYjQ4YWQ2
+    MzJiY2ZhOGY4NDkxNGI2NjU1NmIzZTg0YWVkZmJiMWUxYzU5NWQ=
+  data.tar.gz: !binary |-
+    ZDFmMmE3NWM2YTMzNDNhNjU0ZTdjYWU2YTRmNjZjYmZmMGQ5ZTllYzY2YTdl
+    ZWJiNGY3ZjZlOTM1YmU1ZmYzYzUxZTVjYzE2MGE2OWY0OTA0MGM4YjljZTFi
+    YWNlZWM2NjA0MzA1MjBmYjlhNTMwNDE1OWMzYjhlYzcyMGJhMTI=

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.rspec

@@ -0,0 +1,2 @@
+--format documentation
+--color

data/.travis.yml

@@ -0,0 +1,3 @@
+language: ruby
+rvm:
+  - 1.9.3

data/CHANGELOG

@@ -0,0 +1,3 @@
+0.0.1 / 2014-01-10
+
+Initial release!

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rangefinder.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2014 Seamus Abshere
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,71 @@
+# Rangefinder
+
+Helps you find ranges of IDs, like when you're scraping a website and you need to guess IDs.
+
+You tell it what a valid ID is and it looks for ranges of consecutive valid IDs. It assumes that each probe is expensive.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rangefinder'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rangefinder
+
+## Usage
+
+Let's say you're rainbow tabling a website but you have to guess the IDs. What you **don't** know is that all valid ids are in the ranges `100..11_000` and `100_000..110_000`. You pass a "probe" block that returns true if an ID is valid:
+
+    ranges = Rangefinder.new.probe do |possible_id|
+      # your probe code here. for example:
+      response = http.get "http://example.com/items", id: possible_id
+      response.status == 200
+    end
+
+You get back ranges where we think there are valid IDs. In this case, pretty good! (See Goals above)
+
+    >> ranges
+    => [ 0..12_200, 99_455..111_600 ]
+
+Now you can scrape them one by one:
+
+    ranges.each do |range|
+      range.each do |id|
+        # scrape this ID
+      end
+    end
+
+### Please do cache
+
+It's nice when your probe block makes a call that is cached somehow. That way when you go back and use the ranges, you're not hitting all those URLs over again.
+
+##$ Goals
+
+By default
+
+1. Detect at least 90% of valid IDs in 1000-long ranges with up to 90% intra-range sparsity
+1. Tolerate gaps of 100,000
+1. Probe no more than 5% of the range
+
+Maybe
+
+1. Don't overestimate valid ranges more than X
+
+### Wishlist
+
+1. Accept a known ID as the basis for smarter probing
+1. Internally, calculate density and use that to choose `min_range` and `samp`
+
+## Contributing
+
+1. Fork it ( http://github.com/<my-github-username>/rangefinder/fork )
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,6 @@
+require "bundler/gem_tasks"
+require "rspec/core/rake_task"
+
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/lib/rangefinder.rb

@@ -0,0 +1,58 @@
+require "rangefinder/version"
+require 'rangefinder/memo'
+
+require 'ranges_merger'
+
+class Rangefinder
+  MAX = 2**32 - 1
+  MAX_GAP = 1e5
+  INIT_SAMP = 0.01
+  MAX_SAMP = 0.1
+
+  def probe(options = {}, &blk)
+    ranges, _, _ = probe_with_hits_and_misses(options, &blk)
+  end
+
+  def probe_with_hits_and_misses(options = {}, &blk)
+    memo = Memo.new
+    _probe(memo, options, &blk)
+    [ ::RangesMerger.merge(memo.ranges), memo.hits, memo.misses ]
+  end
+
+  private
+
+  def _probe(memo, options = {}, &blk)
+    first = [options.fetch(:first, 0), 0].max.round
+    last = [options.fetch(:last, MAX), MAX].min.round
+    max_gap = options.fetch(:max_gap, MAX_GAP)
+    samp = options.fetch(:samp, INIT_SAMP)
+    if samp >= MAX_SAMP
+      memo.ranges << (first..last)
+    else
+      min_range = (10 ** (2 - Math.log(samp, 10))).round
+      anything = false
+      first_good = nil
+      i = first
+      last_good = first
+      begin
+        if blk.call(i)
+          memo.hit!
+          anything = true
+          first_good ||= i
+          last_good = i
+        else
+          memo.miss!
+        end
+        gap = i - last_good
+        if first_good and gap > min_range
+          _probe memo, {first: first_good-min_range, last: last_good+min_range, samp: samp*3}, &blk
+          first_good = nil
+          last_good = i
+          gap = 0
+        end
+        samp1 = gap > Math::E ? samp * Math.log(gap) : samp
+        i += (rand(100) * (1 - samp1)).round
+      end until i >= last or (gap > max_gap and anything) # sorry for mixed metaphor
+    end
+  end
+end

data/lib/rangefinder/memo.rb

@@ -0,0 +1,19 @@
+class Rangefinder
+  class Memo
+    attr_reader :ranges
+    attr_reader :hits
+    attr_reader :misses
+    def initialize
+      @ranges = []
+      @hits = 0
+      @misses = 0
+      @mutex = Mutex.new
+    end
+    def hit!
+      @mutex.synchronize { @hits += 1 }
+    end
+    def miss!
+      @mutex.synchronize { @misses += 1 }
+    end
+  end
+end

data/lib/rangefinder/version.rb

@@ -0,0 +1,3 @@
+class Rangefinder
+  VERSION = "0.0.1"
+end

data/rangefinder.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'rangefinder/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "rangefinder"
+  spec.version       = Rangefinder::VERSION
+  spec.authors       = ["Seamus Abshere"]
+  spec.email         = ["seamus@abshere.net"]
+  spec.summary       = %q{Helps you find ranges of IDs, like when you're scraping a website and you need to guess IDs.}
+  spec.description   = %q{Helps you find ranges of IDs, like when you're scraping a website and you need to guess IDs. You tell it what a valid ID is and it looks for ranges of consecutive valid IDs. It assumes that each probe is expensive.}
+  spec.homepage      = "https://github.com/seamusabshere/rangefinder"
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_runtime_dependency 'ranges_merger'
+
+  spec.add_development_dependency "bundler", "~> 1.5"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+  spec.add_development_dependency "pry"
+end

data/spec/rangefinder_spec.rb

@@ -0,0 +1,70 @@
+require 'spec_helper'
+
+# https://github.com/rails/rails/blob/444ce93397dba3505ecef4973edba40de4fc08c6/activesupport/lib/active_support/core_ext/range/include_range.rb#L12
+#  (1..5).include?(1..5) # => true
+#  (1..5).include?(2..3) # => true
+#  (1..5).include?(2..6) # => false
+def range_include?(zelf, other)
+  # 1...10 includes 1..9 but it does not include 1..10.
+  operator = zelf.exclude_end? && !other.exclude_end? ? :< : :<=
+  zelf.include?(other.first) && other.last.send(operator, zelf.last)
+end
+
+describe Rangefinder do
+  expected_ranges = []
+  pos = 0
+  100.times do
+    len = 1000
+    pos += rand(100_000).to_i
+    expected_ranges << ((pos)..(len+pos))
+  end
+  expected_id_count = expected_ranges.map(&:count).inject(:+)
+
+  cache = {}
+
+  (0..0.9).step(0.1).each do |sparsity|
+    describe "sparsity=#{'%g' % sparsity}" do
+      found_ranges, hits, misses = Rangefinder.new.probe_with_hits_and_misses do |i|
+        r = (cache[i] ||= rand)
+        (r > sparsity) && expected_ranges.any? { |r| r.include?(i) }
+      end
+
+      # $stderr.puts
+      # $stderr.puts
+      # $stderr.puts "found_ranges=#{found_ranges}"
+      # $stderr.puts
+      # $stderr.puts "expected_ranges=#{expected_ranges}"
+
+      # it "finds #{expected_ranges.length} ranges" do
+      #   expected_ranges.each do |expected|
+      #     expect(found_ranges.any? { |found| range_include?(found, expected) }).to be_true, "#{expected} not in #{found_ranges} found"
+      #   end
+      # end
+
+      it "finds 95% of ids" do
+        real_found_ids = []
+        expected_ranges.each do |expected|
+          found_ranges.each do |found|
+            # if found.include?(expected)
+            if range_include?(found, expected)
+              real_found_ids << expected.to_a
+            end
+          end
+        end
+        real_found_ids = real_found_ids.flatten.uniq
+        expect((real_found_ids.count.to_f / expected_id_count).round(2)).to be >= 0.95
+      end
+
+      it "probes only 5% of the space" do
+        highest_id = expected_ranges.map(&:last).max
+        expect(((hits+misses).to_f / highest_id).round(2)).to be <= 0.05
+      end
+
+      it "exaggerates no more than 5%" do
+        found_ids = found_ranges.map(&:to_a).flatten.uniq
+        expect((found_ids.count.to_f / expected_id_count).round(2)).to be <= 1.05
+      end
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,4 @@
+require 'pry'
+
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require 'rangefinder'

metadata

@@ -0,0 +1,134 @@
+--- !ruby/object:Gem::Specification
+name: rangefinder
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Seamus Abshere
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-01-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: ranges_merger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.5'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Helps you find ranges of IDs, like when you're scraping a website and
+  you need to guess IDs. You tell it what a valid ID is and it looks for ranges of
+  consecutive valid IDs. It assumes that each probe is expensive.
+email:
+- seamus@abshere.net
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .rspec
+- .travis.yml
+- CHANGELOG
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rangefinder.rb
+- lib/rangefinder/memo.rb
+- lib/rangefinder/version.rb
+- rangefinder.gemspec
+- spec/rangefinder_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/seamusabshere/rangefinder
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.11
+signing_key: 
+specification_version: 4
+summary: Helps you find ranges of IDs, like when you're scraping a website and you
+  need to guess IDs.
+test_files:
+- spec/rangefinder_spec.rb
+- spec/spec_helper.rb
+has_rdoc: