mperham-politics 0.1.0

data/History.rdoc

@@ -0,0 +1,5 @@
+= Changelog
+
+== 0.1.0 (2008-10-07)
+
+* Add BucketWorker and TokenWorker mixins.

data/LICENSE

@@ -0,0 +1,20 @@
+Copyright (c) 2008 Mike Perham
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.rdoc

@@ -0,0 +1,39 @@
+= Politics
+
+Politics is a Ruby library providing utilities and algorithms for solving common distributed
+computing problems.  Distributed Computing and Politics have a number of things in common:
+1) they can be beautiful in theory but get really ugly in reality; 2) after working with
+either for a few weeks/months/years (depending on your moral flexibility) you'll find yourself
+intellectually devoid, a hollow shell of a man/woman/cybernetic killing machine.
+
+So the name is to be taken tongue in cheek.  Onto the real details.
+
+== Common Problems in Distributed Computing
+
+Ruby services are often deployed as a cloud of many processes across several machines, 
+for fault tolerance.  This introduces the problem of coordination between those processes.
+Specifically, how do you keep those processes from stepping on each other's electronic
+toes?  There are several answers:
+
+1. Break the processing into several parts.  Have an individual process "checkout" a part,
+   work on it, and then return the part for processing.  This is a very scalable solution
+   as it allows N workers to work on the same task concurrently.  See the +BucketWorker+ mixin.
+1. Elect a leader for a short period of time.  The leader is the process which performs the
+   actual processing.  After a length of time, a new leader is elected from the group.  This
+   is fault tolerant but not as scalable, as only one process is performing the task at a given
+   point in time.  See the +TokenWorker+ and +PaxosMember+ mixins.
+
+== Dependencies
+
+The BucketWorker mixin uses the Starling queue server as the mechanism to hand out parts.  The
+TokenWorker mixin uses the memcached server as the mechanism to elect a leader.
+
+
+= Author
+
+Name::      Mike Perham
+Email::     mailto:mperham@gmail.com
+Twitter::   http://twitter.com/mperham
+
+This software is free for you to use as you'd like.  If you find it useful, please consider giving 
+me a recommendation at {Working with Rails}[http://workingwithrails.com/person/10797-mike-perham].

data/Rakefile

@@ -0,0 +1,34 @@
+require 'echoe'
+
+require File.dirname(__FILE__) << "/lib/politics/version"
+
+Echoe.new 'politics' do |p|
+  p.version = Politics::Version::STRING
+  p.author = "Mike Perham"
+  p.email  = 'mperham@gmail.com'
+  p.project = 'politics'
+  p.summary = "Algorithms and Tools for Distributed Computing in Ruby."
+  p.url = "http://github.com/mperham/politics"
+  p.dependencies = %w(memcache-client)
+  p.development_dependencies = []
+  p.include_rakefile = true
+  p.rubygems_version = nil
+end
+
+
+require 'rake/testtask'
+
+desc "Run tests"
+Rake::TestTask.new do |t|
+  t.libs << ['test', 'lib']
+	t.test_files = FileList['test/*_test.rb']
+end
+
+desc "Create rdoc"
+Rake::RDocTask.new do |rd|
+	rd.main = "README.rdoc"
+	rd.rdoc_files.include("README.rdoc", "History.rdoc", "lib/**/*.rb")
+end
+
+
+task :default => :test

data/lib/init.rb

@@ -0,0 +1,4 @@
+require 'politics'
+require 'politics/token_worker'
+require 'politics/bucket_worker'
+require 'politics/discoverable_node'

data/lib/politics/bucket_worker.rb

@@ -0,0 +1,107 @@
+begin
+  require 'starling'
+rescue LoadError => e
+  puts "Unable to load starling, please run `sudo gem install starling`: #{e.message}"
+  exit(1)
+end
+
+module Politics
+  
+  # The BucketWorker mixin allows a processing daemon to "lease" or checkout
+  # a portion of a problem space to ensure no other process is processing that same
+  # space at the same time.  The processing space is cut into N "buckets", each of
+  # which is placed in a queue.  Processes then fetch entries from the queue
+  # and process them.  It is up to the application to map the bucket number onto its
+  # specific problem space.
+  #
+  # Note that the Starling queue server is the single point of failure with this
+  # mechanism.  Only you can decide if this is an acceptable tradeoff for your needs.
+  #
+  # Example usage:
+  #
+  #  class Analyzer
+  #    include Politics::BucketWorker
+  #    TOTAL_BUCKETS = 16
+  #
+  #    def start
+  #      register_worker(self.class.name)
+  #      create_buckets(TOTAL_BUCKETS) if master?
+  #      process_bucket do |bucket|
+  #        puts "Analyzing bucket #{bucket} of #{TOTAL_BUCKETS}"
+  #        sleep 5
+  #      end
+  #    end
+  #
+  #    def master?
+  #      # TODO Add your own logic here to denote a 'master' process
+  #      ARGV.include? '-m'
+  #    end
+  #  end
+  #
+  # Note: process_bucket never returns i.e. this should be the main loop of your processing daemon.
+  #
+  module BucketWorker
+    
+    def self.included(model) #:nodoc:
+      model.class_eval do
+        attr_accessor :starling_client, :bucket_count, :queue_name
+      end
+    end
+    
+    # Register this process as able to work on buckets.
+    #
+    # +name+::               The name of the queue to access
+    # +servers+::            The starling server(s) to use, defaults to +['localhost:22122']+
+    def register_worker(name, servers=['localhost:22122'])
+      self.queue_name = name
+      self.starling_client = client_for(Array(servers))
+    end
+    
+    # Create the given number of buckets.  Should ONLY be called by a single worker.
+    # TODO Obviously a major weakness of this algorithm. Is there a cleaner way?
+    #
+    # +bucket_count+::       The number of buckets to create
+    def create_buckets(bucket_count)
+      starling_client.flush(queue_name)
+      bucket_count.times do |count|
+        starling_client.set(queue_name, count)
+      end
+    end
+    
+    # Fetch a bucket out of the queue and pass it to the given block to be processed.
+    # Once processing has completed, it will put the bucket back onto the queue for processing
+    # by a BucketWorker again, possibly immediately, depending on the number of buckets vs
+    # number of workers.
+    #
+    # +bucket+::            The bucket number to process, within the range 0...TOTAL_BUCKETS
+    def process_bucket
+      raise ArgumentError, "process_bucket requires a block!" unless block_given?
+      raise ArgumentError, "You must call register_worker before processing!" unless starling_client
+
+      begin
+        bucket = get_bucket
+        yield bucket
+      ensure
+        push_bucket(bucket)
+      end while loop?
+    end
+    
+    private
+    
+    def get_bucket
+      starling_client.get(queue_name)
+    end
+    
+    def push_bucket(bucket)
+      starling_client.set(queue_name, bucket)
+    end
+    
+    def client_for(servers)
+      Starling.new(servers)
+    end
+    
+    def loop?
+      true
+    end
+  end
+end

data/lib/politics/discoverable_node.rb

@@ -0,0 +1,137 @@
+require 'socket'
+require 'ipaddr'
+require 'uri'
+require 'drb'
+
+require 'net/dns/mdns-sd'
+require 'net/dns/resolv-mdns'
+require 'net/dns/resolv-replace'
+
+=begin
+IRB setup:
+require 'lib/politics'
+require 'lib/politics/discoverable_node'
+require 'lib/politics/convention'
+Object.send(:include, Election::Candidate)
+p = Object.new
+p.register
+=end
+
+module Politics
+
+  # A module to solve the Group Membership problem in distributed computing.
+  # The "group" is the cloud of processes which are replicas and need to coordinate.
+  # Handling group membership is the first step in solving distributed computing
+  # problems.  There are two issues:
+  # 1) replica discovery
+  # 2) controlling and maintaining a consistent group of replicas in each replica
+  #
+  # Peer discovery is implemented using Bonjour for local network auto-discovery.
+  # Each process registers itself on the network as a process of a given type.
+  # Each process then queries the network for other replicas of the same type.
+  #
+  # The replicas then run the Multi-Paxos algorithm to provide consensus on a given
+  # replica set.  The algorithm is robust in the face of crash failures, but not
+  # Byzantine failures.
+	module DiscoverableNode
+	  
+	  attr_accessor :group
+	  attr_accessor :coordinator
+	  
+		def register(group='foo')
+		  self.group = group
+		  start_drb
+		  register_with_bonjour(group)
+		  Politics.log "Registered #{self} in group #{group} with RID #{rid}"
+		  sleep 0.5
+		  find_replicas(0)
+		end
+		
+    def replicas
+      @replicas ||= {}
+    end
+    
+    def find_replicas(count)
+      replicas.clear if count % 5 == 0
+      return if count > 10 # Guaranteed to terminate, but not successfully :-(
+
+      #puts "Finding replicas"
+      peer_set = []
+      bonjour_scan do |replica|
+        (his_rid, his_peers) = replica.hello(rid)
+        unless replicas.has_key?(his_rid)
+          replicas[his_rid] = replica
+        end
+        his_peers.each do |peer|
+          peer_set << peer unless peer_set.include? peer
+        end
+      end
+      #p [peer_set.sort, replicas.keys.sort]
+      if peer_set.sort != replicas.keys.sort
+        # Recursively call ourselves until the network has settled down and all
+        # peers have reached agreement on the peer group membership.
+        sleep 0.2
+        find_replicas(count + 1)
+      end
+      puts "Found #{replicas.size} peers: #{replicas.keys.sort.inspect}" if count == 0
+      replicas
+    end
+    
+    # Called for one peer to introduce itself to another peer.  The caller
+    # sends his RID, the responder sends his RID and his list of current peer
+    # RIDs.
+    def hello(remote_rid)
+      [rid, replicas.keys]
+    end
+    
+	  # A process's Replica ID is its PID + a random 16-bit value.  We don't want
+	  # weigh solely based on PID or IP as that may unduly load one machine.
+	  def rid
+	    @rid ||= begin
+	      rand(65536) + $$
+      end
+    end
+
+		private
+		
+		def register_with_bonjour(group)
+		  # Register our DRb server with Bonjour.
+      handle = Net::DNS::MDNSSD.register("#{self.group}-#{local_ip}-#{$$}", 
+          "_#{self.group}._tcp", 'local', @port)
+          
+      ['INT', 'TERM'].each { |signal| 
+        trap(signal) { handle.stop }
+      }
+	  end
+		
+		def start_drb
+		  server = DRb.start_service(nil, self)
+		  @port = URI.parse(DRb.uri).port
+      ['INT', 'TERM'].each { |signal| 
+        trap(signal) { server.stop_service }
+      }
+	  end
+		
+		def bonjour_scan
+      Net::DNS::MDNSSD.browse("_#{@group}._tcp") do |b|
+        Net::DNS::MDNSSD.resolve(b.name, b.type) do |r|
+          drburl = "druby://#{r.target}:#{r.port}"
+          replica = DRbObject.new(nil, drburl)
+          yield replica
+        end
+      end
+	  end
+	  
+    # http://coderrr.wordpress.com/2008/05/28/get-your-local-ip-address/
+    def local_ip
+      orig, Socket.do_not_reverse_lookup = Socket.do_not_reverse_lookup, true # turn off reverse DNS resolution temporarily
+ 
+      UDPSocket.open do |s|
+        s.connect '64.233.187.99', 1
+        IPAddr.new(s.addr.last).to_i
+      end
+    ensure
+      Socket.do_not_reverse_lookup = orig
+    end	  
+  end
+end

data/lib/politics/token_worker.rb

@@ -0,0 +1,162 @@
+begin
+  require 'memcache'
+rescue LoadError => e
+  puts "Unable to load memcache client, please run `sudo gem install memcache-client`: #{e.message}"
+  exit(1)
+end
+
+module Politics
+
+  # An algorithm to provide leader election between a set of identical processing daemons.
+  #
+  # Each TokenWorker is an instance which needs to perform some processing.
+  # The worker instance must obtain the leader token before performing some task.
+  # We use a memcached server as a central token authority to provide a shared,
+  # network-wide view for all processors.  This reliance on a single resource means
+  # if your memcached server goes down, so do the processors.  Oftentimes, 
+  # this is an acceptable trade-off since many high-traffic web sites would 
+  # not be useable without memcached running anyhow.
+  #
+  # Essentially each TokenWorker attempts to elect itself every +:iteration_length+
+  # seconds by simply setting a key in memcached to its own name.  Memcached tracks
+  # which name got there first.  The key expires after +:iteration_length+ seconds.
+  #
+  # Example usage:
+  #  class Analyzer
+  #    include Politics::TokenWorker
+  #
+  #    def initialize
+  #      register_worker 'analyzer', :iteration_length => 120, :servers => ['localhost:11211']
+  #    end
+  #
+  #    def start
+  #      process do
+  #        # do analysis here, will only be done when this process
+  #        # is actually elected leader, otherwise it will sleep for
+  #        # iteration_length seconds.
+  #      end
+  #    end
+  #  end
+  #
+  # Notes:
+  # * This will not work with multiple instances in the same Ruby process.
+  #   The library is only designed to elect a leader from a set of processes, not instances within
+  #   a single process.
+  # * The algorithm makes no attempt to keep the same leader during the next iteration.
+  #   This can often times be quite beneficial (e.g. leveraging a warm cache from the last iteration)
+  #   for performance but is left to the reader to implement.
+  module TokenWorker
+    
+    def self.included(model) #:nodoc:
+      model.class_eval do
+        attr_accessor :memcache_client, :token, :iteration_length, :worker_name
+        class << self
+          attr_accessor :worker_instance #:nodoc:
+        end
+      end
+    end
+
+    # Register this instance as a worker.
+    #
+    # Options:
+    # +:iteration_length+::    The length of a processing iteration, in seconds.  The 
+    #                          leader's 'reign' lasts for this length of time.
+    # +:servers+::             An array of memcached server strings
+    def register_worker(name, config={})
+      # track the latest instance of this class, there's really only supposed to be
+      # a single TokenWorker instance per process.
+      self.class.worker_instance = self
+
+      options = { :iteration_length => 60, :servers => ['localhost:11211'] }
+      options.merge!(config)
+
+      self.token = "#{name}_token"
+      self.memcache_client = client_for(Array(options[:servers]))
+      self.iteration_length = options[:iteration_length]
+      self.worker_name = "#{Socket.gethostname}:#{$$}"
+
+      cleanup
+    end
+    
+    def process(*args, &block)
+      verify_registration
+
+      begin
+        # Try to add our name as the worker with the master token.
+        # If another process got there first, this is a noop.
+        # We add an expiry so that the master token will constantly
+        # need to be refreshed (in case the current leader dies).
+        nominate
+
+        time = 0
+        if leader?
+          LOG.info { "I've been elected leader: #{worker_name}" }
+          # If we are the master worker, do the work.
+          time = time_for do
+            result = block.call(*args)
+          end
+        end
+        
+        pause_until_expiry(time)
+      end while loop?
+    end
+
+    private
+    
+    def verify_registration
+      unless self.class.worker_instance
+        raise ArgumentError, "Cannot call process without first calling register_worker"
+      end
+      unless self.class.worker_instance == self
+        raise SecurityError, "Only one instance of #{self.class} per process.  Another instance was created after this one."
+      end
+    end
+    
+    def loop?
+      true
+    end
+    
+    def cleanup
+      at_exit do
+        memcache_client.delete(token)
+      end
+    end
+    
+    def pause_until_expiry(elapsed)
+      pause_time = (iteration_length - elapsed).to_f
+      if pause_time > 0
+        relax(pause_time) 
+      else
+        raise ArgumentError, "Negative iteration time left.  Assuming the worst and exiting... #{iteration_length}/#{elapsed}"
+      end
+    end
+    
+    def relax(time)
+      sleep time
+    end
+    
+    # Nominate ourself as leader by contacting the memcached server
+    # and attempting to add the token with our name attached.
+    def nominate
+      memcache_client.add(token, worker_name, iteration_length)
+    end
+
+    # Check to see if we are leader by looking at the process name
+    # associated with the token.
+    def leader?
+      master_worker = memcache_client.get(token)
+      worker_name == master_worker
+    end
+
+    # Easy to mock or monkey-patch if another MemCache client is preferred.
+    def client_for(servers)
+      MemCache.new(servers)
+    end
+
+    def time_for(&block)
+      a = Time.now
+      yield
+      Time.now - a
+    end
+  end
+end

data/lib/politics/version.rb

@@ -0,0 +1,5 @@
+module Politics
+  module Version
+    STRING = "0.1.0"
+  end
+end

data/lib/politics.rb

@@ -0,0 +1,9 @@
+require 'logger'
+
+module Politics
+  LOG = Logger.new(STDOUT)
+  
+  def self.log(msg)
+    LOG.info(msg)
+  end
+end

data/test/bucket_worker_test.rb

@@ -0,0 +1,51 @@
+require 'test_helper'
+
+class BucketWorkerTest < Test::Unit::TestCase
+  
+  context "bucket workers" do
+    setup do
+      @harness = Class.new
+      @harness.send(:include, Politics::BucketWorker)
+      @harness.any_instance.stubs(:loop?).returns(false)
+      @worker = @harness.new
+    end
+
+    should "have instance property accessors" do
+      assert @worker.bucket_count = 20
+      assert_equal 20, @worker.bucket_count
+    end
+
+    should 'register correctly' do
+      @worker.register_worker('testing')
+      @worker.register_worker('testing', ['localhost:5555,localhost:12121'])
+    end
+    
+    should 'process a bucket' do
+      @worker.register_worker('testing')
+      @worker.starling_client.expects(:get).returns(4)
+      @worker.starling_client.expects(:set).with('testing', 4).returns(nil)
+      processed = false
+      @worker.process_bucket do |bucket|
+        assert_equal 4, bucket
+        processed = true
+      end
+      assert processed
+    end
+
+    should 'not allow processing without block' do
+      assert_raises ArgumentError do
+        @worker.register_worker('hello')
+        @worker.process_bucket
+      end
+    end
+
+    should 'not allow processing without registration' do
+      assert_raises ArgumentError do
+        @worker.process_bucket do
+          fail 'Should not process!'
+        end
+      end
+    end
+
+  end
+end

data/test/political_test.rb

@@ -0,0 +1,32 @@
+require 'rubygems'
+require 'test/unit'
+require 'shoulda'
+require File.dirname(__FILE__) + '/../lib/init'
+
+Thread.abort_on_exception = true
+
+class PoliticalTest < Test::Unit::TestCase
+  
+  context "nodes" do
+    setup do
+      @nodes = []
+      5.times do
+        Object.send(:include, Politics::DiscoverableNode)
+        node = Object.new
+        @nodes << node
+      end
+    end
+    
+    should "start up" do
+      processes = @nodes.map do |node|
+        fork do
+          ['INT', 'TERM'].each { |signal| 
+            trap(signal) { exit(0) }
+          }
+          node.register
+        end
+      end
+      Process.wait
+    end
+  end
+end

data/test/test_helper.rb

@@ -0,0 +1,18 @@
+require 'rubygems'
+require 'test/unit'
+
+begin
+  gem 'thoughtbot-shoulda', '>=2.0.2'
+  require 'shoulda'
+rescue GemError, LoadError => e
+  puts "Please install shoulda: `sudo gem install thoughtbot-shoulda -s http://gems.github.com`"
+end
+
+begin
+  require 'mocha'
+rescue LoadError => e
+  puts "Please install mocha: `sudo gem install mocha`"
+end
+
+require File.dirname(__FILE__) + '/../lib/init'
+Politics::LOG.level = Logger::WARN

data/test/token_worker_test.rb

@@ -0,0 +1,67 @@
+require 'test_helper'
+
+class TokenWorkerTest < Test::Unit::TestCase
+  
+  context "token workers" do
+    setup do
+      @harness = Class.new
+      @harness.send(:include, Politics::TokenWorker)
+      @harness.any_instance.stubs(:cleanup)
+      @harness.any_instance.stubs(:loop?).returns(false)
+      @harness.any_instance.stubs(:pause_until_expiry)
+      @harness.any_instance.stubs(:relax)
+
+      @worker = @harness.new
+    end
+
+    should "test_instance_property_accessors" do
+      assert @worker.iteration_length = 20
+      assert_equal 20, @worker.iteration_length
+    end
+
+    should 'test_tracks_a_registered_singleton' do
+      assert_nil @worker.class.worker_instance
+      @worker.register_worker('testing')
+      assert_equal @worker.class.worker_instance, @worker
+    end
+    
+    should 'not process if they are not leader' do
+      @worker.expects(:nominate)
+      @worker.expects(:leader?).returns(false)
+      @worker.register_worker('testing')
+      @worker.process do
+        assert false
+      end
+    end
+
+    should 'process if they are leader' do
+      @worker.expects(:nominate)
+      @worker.expects(:leader?).returns(true)
+      @worker.register_worker('testing')
+
+      worked = 0
+      @worker.process do
+        worked += 1
+      end
+
+      assert_equal 1, worked
+    end
+    
+    should 'not allow processing without registration' do
+      assert_raises ArgumentError do
+        @worker.process
+      end
+    end
+
+    should 'not allow processing by old instances' do
+      @worker.register_worker('testing')
+
+      foo = @worker.class.new
+      foo.register_worker('testing')
+      
+      assert_raises SecurityError do
+        @worker.process
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,93 @@
+--- !ruby/object:Gem::Specification 
+name: mperham-politics
+version: !ruby/object:Gem::Version 
+  version: 0.1.0
+platform: ruby
+authors: 
+- Mike Perham
+autorequire: 
+bindir: bin
+cert_chain: []
+
+date: 2008-08-13 00:00:00 -07:00
+default_executable: 
+dependencies: 
+- !ruby/object:Gem::Dependency 
+  name: fiveruns-memcache-client
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+- !ruby/object:Gem::Dependency 
+  name: fiveruns-starling
+  version_requirement: 
+  version_requirements: !ruby/object:Gem::Requirement 
+    requirements: 
+    - - ">="
+      - !ruby/object:Gem::Version 
+        version: "0"
+    version: 
+description: Algorithms and Tools for Distributed Computing in Ruby.
+email: mperham@gmail.com
+executables: []
+
+extensions: []
+
+extra_rdoc_files: 
+- README.rdoc
+- History.rdoc
+- LICENSE
+files: 
+- README.rdoc
+- LICENSE
+- History.rdoc
+- Rakefile
+- lib/init.rb
+- lib/politics
+- lib/politics/bucket_worker.rb
+- lib/politics/discoverable_node.rb
+- lib/politics/token_worker.rb
+- lib/politics/version.rb
+- lib/politics.rb
+has_rdoc: true
+homepage: http://github.com/mperham/politics/
+post_install_message: 
+rdoc_options: 
+- --quiet
+- --title
+- Politics documentation
+- --opname
+- index.html
+- --line-numbers
+- --main
+- README.rdoc
+- --inline-source
+require_paths: 
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+required_rubygems_version: !ruby/object:Gem::Requirement 
+  requirements: 
+  - - ">="
+    - !ruby/object:Gem::Version 
+      version: "0"
+  version: 
+requirements: []
+
+rubyforge_project: 
+rubygems_version: 1.2.0
+signing_key: 
+specification_version: 2
+summary: Algorithms and Tools for Distributed Computing in Ruby.
+test_files: 
+- test/bucket_worker_test.rb
+- test/political_test.rb
+- test/test_helper.rb
+- test/token_worker_test.rb