mneme 0.5.0

data/.gitignore

@@ -0,0 +1,4 @@
+*.gem
+.bundle
+Gemfile.lock
+pkg/*

data/Gemfile

@@ -0,0 +1,6 @@
+source "http://rubygems.org"
+
+# Specify your gem's dependencies in mneme.gemspec
+gemspec
+
+gem 'redis', :git => 'git://github.com/igrigorik/redis-rb.git'

data/README.md

@@ -0,0 +1,62 @@
+# Mneme
+
+    mneme (n.)  mne·me
+       1. Psychology: the retentive basis or basic principle in a mind or organism accounting for memory, persisting effect of memory of past events.
+       2. Mythology: the Muse of memory, one of the original three Muses. Cf."Aoede, Melete."
+
+Mneme is an HTTP web-service for recording and identifying previously seen records - aka, duplicate detection. To achieve this goal in a scalable, and zero-maintenance manner, it is implemented via a collection of automatically rotated bloomfilters. By using a collection of bloomfilters, you can customize your false-positive error rate, as well as the amount of time you want your memory to perist (ex: remember all keys for the last 6 hours).
+
+To minimize the require memory footprint, mneme does not store the actual key names, instead each specified key is hashed and mapped onto the bloomfilter. For data storage, we use Redis getbit/setbit to efficiently store and retrieve bit-level data for each key. Couple this with Goliath app-server, and you have an out-of-the-box, high-performance, customizable duplicate filter.
+
+For more details: [Mneme: Scalable Duplicate Filtering Service](http://www.igvita.com/2011/03/24/mneme-scalable-duplicate-filtering-service )
+
+## Sample configuration
+
+    # example_config.rb
+
+    config['namespace'] = 'default' # namespace for your app (if you're sharing a redis instance)
+    config['periods'] = 3           # number of periods to store data for
+    config['length']  = 60          # length of a period in seconds (length = 60, periods = 3.. 180s worth of data)
+
+    config['size']    = 1000        # desired size of the bloomfilter
+    config['bits']    = 10          # number of bits allocated per key
+    config['hashes']  = 7           # number of times each key will be hashed
+    config['seed']    = 30          # seed value for the hash function
+
+To learn more about Bloom filter configuration: [Scalable Datasets: Bloom Filters in Ruby](http://www.igvita.com/2008/12/27/scalable-datasets-bloom-filters-in-ruby/)
+
+## Launching mneme
+
+    $> redis-server
+    $> gem install mneme
+    $> mneme -p 9000 -sv -c config.rb   # run with -h to see all options
+
+That's it! You now have a mneme web service running on port 9000. Let's try querying and inserting some data:
+
+    $> curl "http://127.0.0.1:9000?key=abcd"
+    {"found":[],"missing":["abcd"]}
+
+    # -d creates a POST request with key=abcd, aka insert into filter
+    $> curl "http://127.0.0.1:9000?key=abcd" -d' '
+
+    $> curl "http://127.0.0.1:9000?key=abcd"
+    {"found":["abcd"],"missing":[]}
+
+## Performance & Memory requirements
+
+ - [Redis](http://redis.io/) is used as an in-memory datastore of the bloomfilter
+ - [Goliath](https://github.com/postrank-labs/goliath) provides the high-performance HTTP frontend
+ - The speed of storing a new key is: *O(number of BF hashes) - aka, O(1)*
+ - The speed of retrieving a key is: *O(number of filters * number of BF hashes) - aka, O(1)*
+
+Bloom filter is a space-efficient probabilistic data structure that is used to test whether an element is a member of a set. False positives are possible, but false negatives are not. Because we are using Redis as a backend, in-memory store for the filters, there is some extra overhead. Sample memory requirements:
+
+ - 1.0% error rate for 1M items, 10 bits/item: 2.5 mb
+ - 1.0% error rate for 150M items, 10 bits per item: 358.52 mb
+ - 0.1% error rate for 150M items, 15 bits per item: 537.33 mb
+
+Ex: If you wanted to store up to 24 hours (with 1 hour = 1 bloom filter) of keys, where each hour can have up to 1M keys, and you are willing to accept a 1.0% error rate, then your memory footprint is: 24 * 2.5mb = 60mb of memory. The footprint will not change after 24 hours, because Mneme will automatically rotate and delete old filters for you!
+
+### License
+
+(MIT License) - Copyright (c) 2011 Ilya Grigorik

data/Rakefile

@@ -0,0 +1,9 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+require 'rspec/core/rake_task'
+
+desc "Run all RSpec tests"
+RSpec::Core::RakeTask.new(:spec)
+
+task :default => :spec

data/bin/mneme

@@ -0,0 +1,11 @@
+#!/usr/bin/env ruby
+
+begin
+  config_index = (ARGV.index('-c') || ARGV.index('--config')) + 1
+  ARGV[config_index] = File.absolute_path(ARGV[config_index])
+rescue
+  puts "Please specify a valid mneme configuration file (ex: -c config.rb)"
+  exit
+end
+
+system("/usr/bin/env ruby " + File.dirname(__FILE__) + '/../lib/mneme.rb' + ' ' + ARGV.join(" "))

data/config.rb

@@ -0,0 +1,8 @@
+config['namespace'] = 'default'
+config['periods'] = 3
+config['length']  = 60
+
+config['size']    = 1000
+config['bits']    = 10
+config['hashes']  = 7
+config['seed']    = 30

data/lib/mneme.rb

@@ -0,0 +1,97 @@
+require 'goliath'
+require 'yajl'
+
+require 'redis'
+require 'redis/connection/synchrony'
+require 'bloomfilter-rb'
+
+require 'lib/mneme/helper'
+require 'lib/mneme/sweeper'
+
+class Mneme < Goliath::API
+  include Mnemosyne::Helper
+  plugin Mnemosyne::Sweeper
+
+  use ::Rack::Reloader, 0 if Goliath.dev?
+
+  use Goliath::Rack::Params
+  use Goliath::Rack::DefaultMimeType
+  use Goliath::Rack::Formatters::JSON
+  use Goliath::Rack::Render
+  use Goliath::Rack::Heartbeat
+  use Goliath::Rack::ValidationError
+  use Goliath::Rack::Validation::RequestMethod, %w(GET POST)
+
+  def response(env)
+    keys = [params.delete('key') || params.delete('key[]')].flatten.compact
+    return [400, {}, {error: 'no key specified'}] if keys.empty?
+
+    logger.debug "Processing: #{keys}"
+    case env[Goliath::Request::REQUEST_METHOD]
+      when 'GET'  then query_filters(keys)
+      when 'POST' then update_filters(keys)
+    end
+  end
+
+  def query_filters(keys)
+    found, missing = [], []
+    keys.each do |key|
+
+      present = false
+      config['periods'].to_i.times do |n|
+        if filter(n).key?(key)
+          present = true
+          break
+        end
+      end
+
+      if present
+        found << key
+      else
+        missing << key
+      end
+    end
+
+    code = case keys.size
+      when found.size then 200
+      when missing.size then 404
+      else 206
+    end
+
+    [code, {}, {found: found, missing: missing}]
+  end
+
+  def update_filters(keys)
+    keys.each do |key|
+      filter(0).insert key
+      logger.debug "Inserted new key: #{key}"
+    end
+
+    [201, {}, '']
+  end
+
+  private
+
+    def filter(n)
+      period = epoch_name(config['namespace'], n, config['length'])
+
+      filter = if env.key? period
+        env[period]
+      else
+        opts = {
+          namespace: config['namespace'],
+          size: config['size'] * config['bits'],
+          seed: config['seed'],
+          hashes: config['hashes']
+        }
+
+        # env[period] = EventMachine::Synchrony::ConnectionPool.new(size: 10) do
+        env[period] = BloomFilter::Redis.new(opts)
+        # end
+
+        env[period]
+      end
+
+      filter
+    end
+end

data/lib/mneme/helper.rb

@@ -0,0 +1,11 @@
+module Mnemosyne
+  module Helper
+    def epoch(n, length)
+      (Time.now.to_i / length) - n
+    end
+
+    def epoch_name(namespace, n, length)
+      "mneme-#{namespace}-#{epoch(n, length)}"
+    end
+  end
+end

data/lib/mneme/sweeper.rb

@@ -0,0 +1,38 @@
+module Mnemosyne
+  class Sweeper
+    include Helper
+
+    def initialize(port, config, status, logger)
+      @status = status
+      @config = config
+      @logger = logger
+    end
+
+    def run
+      if @config.empty?
+        puts "Please specify a valid mneme configuration file (ex: -c config.rb)"
+        EM.stop
+        exit
+      end
+
+      sweeper = Proc.new do
+        current = epoch_name(@config['namespace'], 0, @config['length'])
+        @logger.info "Sweeping old filters, current epoch: #{current}"
+
+        conn = Redis.new
+        @config['periods'].times do |n|
+          name = epoch_name(@config['namespace'], n + @config['periods'], @config['length'])
+
+          conn.del(name)
+          @logger.info "Removed: #{name}"
+        end
+        conn.client.disconnect
+      end
+
+      sweeper.call
+      EM.add_periodic_timer(@config['length']) { sweeper.call }
+
+      @logger.info "Started Mnemosyne::Sweeper with #{@config['length']}s interval"
+    end
+  end
+end

data/mneme.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+
+Gem::Specification.new do |s|
+  s.name        = "mneme"
+  s.version     = "0.5.0"
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = ""
+  s.summary     = %q{abc}
+  s.description = %q{Write a gem description}
+
+  s.rubyforge_project = "mneme"
+
+  s.add_dependency "goliath"
+  s.add_dependency "hiredis"
+
+  s.add_dependency "redis"
+  s.add_dependency "yajl-ruby"
+  s.add_dependency "bloomfilter-rb"
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "em-http-request", ">= 1.0.0.beta.3"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/spec/mneme_spec.rb

@@ -0,0 +1,93 @@
+require 'lib/mneme'
+require 'goliath/test_helper'
+require 'em-http/middleware/json_response'
+
+describe Mneme do
+  include Goliath::TestHelper
+
+  let(:err) { Proc.new { fail "API request failed" } }
+  let(:api_options) { { :config => File.expand_path(File.join(File.dirname(__FILE__), '..', 'config.rb')) } }
+
+  EventMachine::HttpRequest.use EventMachine::Middleware::JSONResponse
+
+  it 'responds to hearbeat' do
+    with_api(Mneme, api_options) do
+      get_request({path: '/status'}, err) do |c|
+        c.response.should match('OK')
+      end
+    end
+  end
+
+  it 'should require an error if no key is provided' do
+    with_api(Mneme, api_options) do
+      get_request({}, err) do |c|
+        c.response.should include 'error'
+      end
+    end
+  end
+
+  context 'single key' do
+    it 'should return 404 on missing key' do
+      with_api(Mneme, api_options) do
+        get_request({:query => {:key => 'missing'}}, err) do |c|
+          c.response_header.status.should == 404
+          c.response['missing'].should include 'missing'
+        end
+      end
+    end
+
+    it 'should insert key into filter' do
+      with_api(Mneme, api_options) do
+        post_request({:query => {key: 'abc'}}) do |c|
+          c.response_header.status.should == 201
+
+          get_request({:query => {:key => 'abc'}}, err) do |c|
+            c.response_header.status.should == 200
+            c.response['found'].should include 'abc'
+          end
+        end
+      end
+    end
+  end
+
+  context 'multiple keys' do
+
+    it 'should return 404 on missing keys' do
+      with_api(Mneme, api_options) do
+        get_request({:query => {:key => ['a', 'b']}}, err) do |c|
+          c.response_header.status.should == 404
+
+          c.response['found'].should be_empty
+          c.response['missing'].should include 'a'
+          c.response['missing'].should include 'b'
+        end
+      end
+    end
+
+    it 'should return 200 on found keys' do
+      with_api(Mneme, api_options) do
+        post_request({:query => {key: ['abc1', 'abc2']}}) do |c|
+          c.response_header.status.should == 201
+
+          get_request({:query => {:key => ['abc1', 'abc2']}}, err) do |c|
+            c.response_header.status.should == 200
+          end
+        end
+      end
+    end
+
+    it 'should return 206 on mixed keys' do
+      with_api(Mneme, api_options) do
+        post_request({:query => {key: ['abc3']}}) do |c|
+          c.response_header.status.should == 201
+
+          get_request({:query => {:key => ['abc3', 'abc4']}}, err) do |c|
+            c.response_header.status.should == 206
+          end
+        end
+      end
+    end
+
+  end
+
+end

metadata

@@ -0,0 +1,144 @@
+--- !ruby/object:Gem::Specification 
+name: mneme
+version: !ruby/object:Gem::Version 
+  prerelease: 
+  version: 0.5.0
+platform: ruby
+authors: 
+- Ilya Grigorik
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2011-03-24 00:00:00 -04:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: goliath
+  prerelease: false
+  requirement: &id001 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id001
+- !ruby/object:Gem::Dependency 
+  name: hiredis
+  prerelease: false
+  requirement: &id002 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id002
+- !ruby/object:Gem::Dependency 
+  name: redis
+  prerelease: false
+  requirement: &id003 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id003
+- !ruby/object:Gem::Dependency 
+  name: yajl-ruby
+  prerelease: false
+  requirement: &id004 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency 
+  name: bloomfilter-rb
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :runtime
+  version_requirements: *id005
+- !ruby/object:Gem::Dependency 
+  name: rspec
+  prerelease: false
+  requirement: &id006 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+  type: :development
+  version_requirements: *id006
+- !ruby/object:Gem::Dependency 
+  name: em-http-request
+  prerelease: false
+  requirement: &id007 !ruby/object:Gem::Requirement 
+    none: false
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: 1.0.0.beta.3
+  type: :development
+  version_requirements: *id007
+description: Write a gem description
+email: 
+- ilya@igvita.com
+executables: 
+- mneme
+extensions: []
+
+extra_rdoc_files: []
+
+files: 
+- .gitignore
+- .rspec
+- Gemfile
+- README.md
+- Rakefile
+- bin/mneme
+- config.rb
+- lib/mneme.rb
+- lib/mneme/helper.rb
+- lib/mneme/sweeper.rb
+- mneme.gemspec
+- spec/mneme_spec.rb
+has_rdoc: true
+homepage: ""
+licenses: []
+
+post_install_message: 
+rdoc_options: []
+
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  none: false
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+requirements: []
+
+rubyforge_project: mneme
+rubygems_version: 1.6.2
+signing_key: 
+specification_version: 3
+summary: abc
+test_files: 
+- spec/mneme_spec.rb