twirl 0.1.0

data/lib/twirl/version.rb

@@ -0,0 +1,3 @@
+module Twirl
+  VERSION = "0.1.0"
+end

data/script/bootstrap

@@ -0,0 +1,21 @@
+#!/bin/sh
+#/ Usage: bootstrap [bundle options]
+#/
+#/ Bundle install the dependencies.
+#/
+#/ Examples:
+#/
+#/   bootstrap
+#/   bootstrap --local
+#/
+
+set -e
+cd $(dirname "$0")/..
+
+[ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && {
+    grep '^#/' <"$0"| cut -c4-
+    exit 0
+}
+
+rm -rf .bundle/{binstubs,config}
+bundle install --binstubs .bundle/binstubs --path .bundle "$@"

data/script/kestrel

@@ -0,0 +1,34 @@
+#!/usr/bin/env ruby
+
+require "rubygems"
+require "bundler/setup"
+require "pathname"
+root = Pathname(__FILE__).join("..", "..").expand_path
+
+$LOAD_PATH.unshift root.join("lib")
+require "twirl/server"
+
+{
+  "kestrel1" => {
+    memcache_port: 9444,
+    thrift_port: 9445,
+    text_port: 9446,
+    admin_port: 9447,
+  },
+  "kestrel2" => {
+    memcache_port: 9544,
+    thrift_port: 9545,
+    text_port: 9546,
+    admin_port: 9547,
+  },
+}.each do |dir, options|
+  server = Twirl::Server.new(root.join("tmp", dir), options)
+  puts "\n\nMaking sure kestrel is running on the following ports: #{options.inspect}"
+  if ARGV[0] == "stop"
+    server.stop
+  else
+    server.start
+  end
+end
+
+puts "\n\nYou can now run the tests."

data/script/release

@@ -0,0 +1,42 @@
+#!/bin/sh
+#/ Usage: release
+#/
+#/ Tag the version in the repo and push the gem.
+#/
+
+set -e
+cd $(dirname "$0")/..
+
+[ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && {
+    grep '^#/' <"$0"| cut -c4-
+    exit 0
+}
+
+gem_name=twirl
+
+# Build a new gem archive.
+rm -rf $gem_name-*.gem
+gem build -q $gem_name.gemspec
+
+# Make sure we're on the master branch.
+(git branch | grep -q '* master') || {
+  echo "Only release from the master branch."
+  exit 1
+}
+
+# Figure out what version we're releasing.
+tag=v`ls $gem_name-*.gem | sed "s/^$gem_name-\(.*\)\.gem$/\1/"`
+
+echo "Releasing $tag"
+
+# Make sure we haven't released this version before.
+git fetch -t origin
+
+(git tag -l | grep -q "$tag") && {
+  echo "Whoops, there's already a '${tag}' tag."
+  exit 1
+}
+
+# Tag it and bag it.
+gem push $gem_name-*.gem && git tag "$tag" &&
+  git push origin master && git push origin "$tag"

data/script/test

@@ -0,0 +1,25 @@
+#!/bin/sh
+#/ Usage: test [individual test file]
+#/
+#/ Bootstrap and run all tests or an individual test.
+#/
+#/ Examples:
+#/
+#/   # run all tests
+#/   test
+#/
+#/   # run individual test
+#/   test test/twirl_test.rb
+#/
+
+set -e
+cd $(dirname "$0")/..
+
+[ "$1" = "--help" -o "$1" = "-h" -o "$1" = "help" ] && {
+    grep '^#/' <"$0"| cut -c4-
+    exit 0
+}
+
+script/bootstrap && ruby -I lib -I test -r rubygems \
+  -e 'require "bundler/setup"' \
+  -e '(ARGV.empty? ? Dir["test/**/*_test.rb"] : ARGV).each { |f| load f }' -- "$@"

data/script/watch

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+#/ Usage: watch
+#/
+#/ Run the tests whenever any relevant files change.
+#/
+
+require "pathname"
+require "rubygems"
+require "bundler/setup"
+
+# Put us where we belong, in the root dir of the project.
+Dir.chdir Pathname.new(__FILE__).realpath + "../.."
+
+# Run the tests to start.
+system "clear; script/test"
+
+require "rb-fsevent"
+
+IgnoreRegex = /\/fixtures/
+
+fs = FSEvent.new
+fs.watch ["app", "lib", "test"], latency: 1 do |args|
+  unless args.first =~ IgnoreRegex
+    system "clear"
+    puts "#{args.first} changed..."
+    system "script/test"
+  end
+end
+fs.run

data/test/cluster_test.rb

@@ -0,0 +1,469 @@
+require "helper"
+require "twirl/cluster"
+
+class ClusterTest < Minitest::Test
+  def test_initialize
+    clients = [
+      KJess::Client.new(host: "localhost", port: 1),
+      KJess::Client.new(host: "localhost", port: 2),
+    ]
+    cluster = Twirl::Cluster.new(clients)
+    assert_equal 0, cluster.command_count
+    assert_equal 0, cluster.client_index
+    assert_equal 100, cluster.commands_per_client
+    assert_equal 5, cluster.retries
+    assert_equal [KJess::NetworkError, KJess::ServerError],
+      cluster.retryable_errors
+    assert_equal Twirl::Instrumenters::Noop, cluster.instrumenter
+  end
+
+  def test_initialize_shuffles_clients
+    clients = Minitest::Mock.new
+    clients.expect :shuffle, :shuffle_result
+    cluster = Twirl::Cluster.new(clients)
+    assert_equal :shuffle_result, cluster.instance_variable_get("@clients")
+    clients.verify
+  end
+
+  def test_overriding_instrumenter
+    instrumenter = Object.new
+    cluster = Twirl::Cluster.new([], instrumenter: instrumenter)
+    assert_equal instrumenter, cluster.instrumenter
+  end
+
+  def test_overriding_retryable_errors
+    retryable_errors = StandardError
+    cluster = Twirl::Cluster.new([], retryable_errors: retryable_errors)
+    assert_equal retryable_errors, cluster.retryable_errors
+  end
+
+  def test_overriding_retries
+    cluster = Twirl::Cluster.new([], retries: 1)
+    assert_equal 1, cluster.retries
+  end
+
+  def test_enumeration
+    cluster = build(:mock_cluster)
+    clients = []
+    cluster.each_with_index do |client, index|
+      client.expect :port, index
+      clients << client
+    end
+    assert_equal clients, cluster.to_a
+    assert_equal [0, 1, 2], cluster.map(&:port)
+  end
+
+  def test_size
+    cluster = build(:mock_cluster)
+    assert_equal 3, cluster.size
+    assert_equal 3, cluster.length
+    assert_equal 3, cluster.count
+  end
+
+  def test_bracket_access
+    clients = [
+      KJess::Client.new(host: "localhost", port: 1),
+      KJess::Client.new(host: "localhost", port: 2),
+      KJess::Client.new(host: "localhost", port: 3),
+    ]
+    cluster = Twirl::Cluster.new(clients)
+    shuffled_clients = cluster.instance_variable_get("@clients")
+    assert_equal shuffled_clients[0], cluster[0]
+    assert_equal shuffled_clients[1], cluster[1]
+    assert_equal shuffled_clients[2], cluster[2]
+  end
+
+  def test_set_rotates_based_on_commands_per_client
+    args = ["testing", "data", 0]
+    cluster = build(:mock_cluster, commands_per_client: 1)
+
+    cluster.each do |client|
+      client.expect :set, true, args
+    end
+
+    assert_equal true, cluster.set(*args)
+    assert_equal true, cluster.set(*args)
+    assert_equal true, cluster.set(*args)
+
+    cluster.each(&:verify)
+  end
+
+  def test_get_returns_item_if_client_returns_item
+    args = ["testing", {}]
+    cluster = build(:mock_cluster)
+    cluster[0].expect :get, "data", args
+
+    item = cluster.get(*args)
+    assert_instance_of Twirl::Item, item
+    assert_equal "testing", item.key
+    assert_equal "data", item.value
+
+    cluster.each(&:verify)
+  end
+
+  def test_get_returns_nil_if_client_returns_nil
+    args = ["testing", {}]
+    cluster = build(:mock_cluster)
+    cluster[0].expect :get, nil, args
+
+    assert_nil cluster.get(*args)
+
+    cluster.each(&:verify)
+  end
+
+  def test_get_passes_options_to_client
+    args = ["testing", {open: true}]
+    cluster = build(:mock_cluster)
+    cluster[0].expect :get, nil, args
+    cluster.get(*args)
+    cluster.each(&:verify)
+  end
+
+  def test_get_rotates_based_on_commands_per_client
+    args = ["testing", {}]
+    cluster = build(:mock_cluster, commands_per_client: 1)
+
+    cluster[0].expect :get, "result-1", args
+    cluster[1].expect :get, "result-2", args
+    cluster[2].expect :get, "result-3", args
+
+    assert_equal "result-1", cluster.get(*args).value
+    assert_equal "result-2", cluster.get(*args).value
+    assert_equal "result-3", cluster.get(*args).value
+
+    cluster.each(&:verify)
+  end
+
+  def test_get_rotates_client_if_nil
+    args = ["testing", {}]
+    cluster = build(:mock_cluster, commands_per_client: 1_000)
+
+    cluster[0].expect :get, nil, args
+    cluster[1].expect :get, nil, args
+    cluster[2].expect :get, "data", args
+
+    assert_nil cluster.get(*args)
+    assert_nil cluster.get(*args)
+    assert_equal "data", cluster.get(*args).value
+
+    cluster.each(&:verify)
+  end
+
+  def test_reserve
+    cluster = build(:mock_cluster)
+    cluster[0].expect :reserve, "data", ["testing", {}]
+    assert_equal "data", cluster.reserve("testing").value
+    cluster.each(&:verify)
+  end
+
+  def test_reserve_returns_nil_if_client_returns_nil
+    cluster = build(:mock_cluster)
+    cluster[0].expect :reserve, nil, ["testing", {}]
+    assert_nil cluster.reserve("testing")
+    cluster.each(&:verify)
+  end
+
+  def test_reserve_rotates_client_if_nil
+    args = ["testing", {}]
+    cluster = build(:mock_cluster, commands_per_client: 1_000)
+
+    cluster[0].expect :reserve, nil, args
+    cluster[1].expect :reserve, nil, args
+    cluster[2].expect :reserve, "data", args
+
+    assert_nil cluster.reserve(*args)
+    assert_nil cluster.reserve(*args)
+    assert_equal "data", cluster.reserve(*args).value
+
+    cluster.each(&:verify)
+  end
+
+  def test_peek
+    cluster = build(:mock_cluster)
+    cluster[0].expect :peek, "data", ["testing"]
+    assert_equal "data", cluster.peek("testing").value
+    cluster.each(&:verify)
+  end
+
+  def test_peek_returns_nil_if_client_returns_nil
+    cluster = build(:mock_cluster)
+    cluster[0].expect :peek, nil, ["testing"]
+    assert_nil cluster.peek("testing")
+    cluster.each(&:verify)
+  end
+
+  def test_peek_rotates_client_if_nil
+    args = ["testing"]
+    cluster = build(:mock_cluster, commands_per_client: 1_000)
+
+    cluster[0].expect :peek, nil, args
+    cluster[1].expect :peek, nil, args
+    cluster[2].expect :peek, "data", args
+
+    assert_nil cluster.peek(*args)
+    assert_nil cluster.peek(*args)
+    assert_equal "data", cluster.peek(*args).value
+
+    cluster.each(&:verify)
+  end
+
+  def test_flush
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :flush, index % 2 == 0, ["foo"]
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.flush("foo")
+    cluster.each(&:verify)
+  end
+
+  def test_flush_all
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :flush_all, index % 2 == 0
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.flush_all
+    cluster.each(&:verify)
+  end
+
+  def test_disconnect
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :disconnect, nil
+    end
+    cluster.disconnect
+    cluster.each(&:verify)
+  end
+
+  def test_version
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :version, "2.4.1"
+    end
+    expected = {
+      "localhost:1" => "2.4.1",
+      "localhost:2" => "2.4.1",
+      "localhost:3" => "2.4.1",
+    }
+    assert_equal expected, cluster.version
+    cluster.each(&:verify)
+  end
+
+  def test_version_with_one_raising_exception
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+    end
+
+    # force client1 to raise an error and other clients to return as normal
+    client1 = cluster[0]
+    def client1.version
+      raise KJess::ProtocolError
+    end
+    cluster[1].expect :version, "2.4.1"
+    cluster[2].expect :version, "2.4.1"
+
+    expected = {
+      "localhost:1" => "unavailable",
+      "localhost:2" => "2.4.1",
+      "localhost:3" => "2.4.1",
+    }
+    assert_equal expected, cluster.version
+    cluster.each(&:verify)
+  end
+
+  def test_delete
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :delete, index % 2 == 0, ["foo"]
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.delete("foo")
+    cluster.each(&:verify)
+  end
+
+  def test_ping
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :ping, index % 2 == 0
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.ping
+    cluster.each(&:verify)
+  end
+
+  def test_shutdown
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :shutdown, nil
+    end
+    cluster.shutdown
+    cluster.each(&:verify)
+  end
+
+  def test_reload
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :reload, index % 2 == 0
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.reload
+    cluster.each(&:verify)
+  end
+
+  def test_quit
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :quit, index % 2 == 0
+    end
+    expected = {
+      "localhost:1" => true,
+      "localhost:2" => false,
+      "localhost:3" => true,
+    }
+    assert_equal expected, cluster.quit
+    cluster.each(&:verify)
+  end
+
+  def test_stats
+    cluster = build(:mock_cluster)
+    cluster.size.times do |index|
+      cluster[index].expect :host, "localhost"
+      cluster[index].expect :port, (index + 1).to_s
+      cluster[index].expect :stats, {items: index + 1}
+    end
+    expected = {
+      "localhost:1" => {items: 1},
+      "localhost:2" => {items: 2},
+      "localhost:3" => {items: 3},
+    }
+    assert_equal expected, cluster.stats
+    cluster.each(&:verify)
+  end
+
+  def test_rotation
+    cluster = build(:mock_cluster, commands_per_client: 3)
+    assert_equal 0, cluster.command_count
+
+    counts = 7.times.map {
+      cluster.client
+      cluster.command_count
+    }
+
+    assert_equal [1, 2, 3, 1, 2, 3, 1], counts
+  end
+
+  RetriableMethods = {
+    get: [["testing"], "data"],
+    reserve: [["testing"], "data"],
+    peek: [["testing"], "data"],
+    set: [["testing", "data"], true],
+  }
+
+  RecoverableExceptions = [
+    KJess::NetworkError,
+    KJess::ServerError,
+  ]
+
+  def test_get_retries_network_errors
+    cluster = build(:cluster)
+    client = cluster[0]
+
+    RetriableMethods.each do |op, info|
+      args, result = info
+      RecoverableExceptions.each do |exception|
+        client.instance_eval <<-EOC
+          def client.#{op}(*args)
+            @counter ||= 0
+            if @counter < 4
+              @counter += 1
+              raise #{exception}
+            else
+              #{result.inspect}
+            end
+          end
+
+          def client.reset
+            @counter = 0
+          end
+        EOC
+
+        begin
+          op_result = cluster.send(op, *args)
+        ensure
+          client.reset
+        end
+      end
+    end
+  end
+
+  def test_too_many_retries_raises
+    cluster = build(:cluster)
+    client = cluster[0]
+
+    RetriableMethods.each do |op, info|
+      args, result = info
+      RecoverableExceptions.each do |exception|
+        client.instance_eval <<-EOC
+          def client.#{op}(*args)
+            @counter ||= 0
+            if @counter < 5
+              @counter += 1
+              raise #{exception}
+            else
+              #{result.inspect}
+            end
+          end
+
+          def client.reset
+            @counter = 0
+          end
+        EOC
+
+        assert_raises exception, "#{op} raising #{exception}" do
+          begin
+            cluster.send(op, *args)
+          ensure
+            client.reset
+          end
+        end
+      end
+    end
+  end
+end