twirl 0.1.0

data/test/helper.rb

@@ -0,0 +1,65 @@
+require "minitest/autorun"
+require_relative "support/fake_udp_socket"
+require "twirl/instrumenters/noop"
+
+module ClusterTestHelpers
+  def build(what, *args)
+    send "build_#{what}", *args
+  end
+
+  private
+
+  def build_item(key, value, client, instrumenter = nil)
+    Twirl::Item.new(key, value, client, instrumenter)
+  end
+
+  # Builds a cluster that will connect to what script/kestrel starts.
+  def build_cluster(options = {})
+    clients = [
+      KJess::Client.new(host: "localhost", port: 9444),
+      KJess::Client.new(host: "localhost", port: 9544),
+    ]
+    queue_names = Array(options.delete(:queues))
+    Twirl::Cluster.new(clients, options).tap { |cluster|
+      queue_names.each { |queue_name|
+        cluster.each { |client|
+          client.flush queue_name
+        }
+      }
+    }
+  end
+
+  def build_mock_cluster(options = {})
+    number_of_clients = options.delete(:number_of_clients) || 3
+    clients = number_of_clients.times.map { Minitest::Mock.new }
+
+    Twirl::Cluster.new clients, options
+  end
+end
+
+module StatsdTestHelpers
+  def assert_timer(socket, metric)
+    regex = /#{Regexp.escape metric}\:\d+\|ms/
+    assert socket.buffer.detect { |op| op.first =~ regex },
+      "Timer #{metric} was not found in #{socket.buffer.inspect}"
+  end
+
+  def assert_counter(socket, metric)
+    assert socket.buffer.detect { |op| op.first == "#{metric}:1|c" },
+      "Counter #{metric} was not found in #{socket.buffer.inspect}"
+  end
+
+  def assert_no_timer(socket, metric)
+    regex = /#{Regexp.escape metric}\:\d+\|ms/
+    assert_nil socket.buffer.detect { |op| op.first =~ regex },
+      "Timer #{metric} was found in #{socket.buffer.inspect}"
+  end
+
+  def assert_no_counter(socket, metric)
+    assert_nil socket.buffer.detect { |op| op.first == "#{metric}:1|c" },
+      "Counter #{metric} was found in #{socket.buffer.inspect}"
+  end
+end
+
+Minitest::Test.send :include, ClusterTestHelpers
+Minitest::Test.send :include, StatsdTestHelpers

data/test/instrumentation/log_subscriber_test.rb

@@ -0,0 +1,51 @@
+require "helper"
+require "logger"
+require "twirl/instrumentation/log_subscriber"
+
+class LogSubscriberInstrumentationTest < Minitest::Test
+  def setup
+    @cluster = build(:cluster, {
+      instrumenter: ActiveSupport::Notifications,
+      queues: %w(testing),
+    })
+    @io = StringIO.new
+    logger = Logger.new(@io)
+    logger.formatter = proc { |severity, datetime, progname, msg| "#{msg}\n" }
+    Twirl::Instrumentation::LogSubscriber.logger = logger
+  end
+
+  def log
+    @io.string
+  end
+
+  def teardown
+    Twirl::Instrumentation::LogSubscriber.logger = nil
+  end
+
+  def test_set
+    @cluster.set "testing", "data"
+    line = find_line("Twirl op(set)")
+    assert_match %r{queue_name=testing bytes=4}, line
+  end
+
+  def test_get
+    @cluster.set "testing", "data"
+    @cluster.get "testing"
+    line = find_line("Twirl op(get)")
+    assert_match %r{queue_name=testing bytes=4}, line
+  end
+
+  def test_reserve
+    @cluster.set "testing", "data"
+    @cluster.reserve "testing"
+    line = find_line("Twirl op(reserve)")
+    assert_match %r{queue_name=testing bytes=4}, line
+  end
+
+  def find_line(str)
+    regex = /#{Regexp.escape(str)}/
+    lines = log.split("\n")
+    lines.detect { |line| line =~ regex } ||
+      raise("Could not find line matching #{str.inspect} in #{lines.inspect}")
+  end
+end

data/test/instrumentation/statsd_test.rb

@@ -0,0 +1,173 @@
+require "helper"
+require "twirl/instrumentation/statsd"
+
+class StatsdInstrumentationTest < Minitest::Test
+  def setup
+    @statsd = Statsd.new
+    @socket = FakeUDPSocket.new
+    @cluster = build(:cluster, {
+      instrumenter: ActiveSupport::Notifications,
+      queues: %w(testing),
+    })
+    Thread.current[:statsd_socket] = @socket
+    Twirl::Instrumentation::StatsdSubscriber.client = @statsd
+  end
+
+  def teardown
+    Thread.current[:statsd_socket] = nil
+    Twirl::Instrumentation::StatsdSubscriber.client = nil
+  end
+
+  def test_set
+    @cluster.set "testing", "data"
+    assert_timer @socket, "twirl.op_set"
+    assert_timer @socket, "twirl.queue_testing_op_set"
+    assert_counter @socket, "twirl.bytes_op_set"
+    assert_counter @socket, "twirl.bytes_queue_testing_op_set"
+  end
+
+  def test_get
+    @cluster.set "testing", "data"
+    @cluster.get "testing"
+    assert_timer @socket, "twirl.op_get"
+    assert_timer @socket, "twirl.queue_testing_op_get"
+    assert_counter @socket, "twirl.bytes_op_get"
+    assert_counter @socket, "twirl.bytes_queue_testing_op_get"
+  end
+
+  def test_item_close
+    @cluster.set "testing", "data"
+    item = @cluster.reserve("testing")
+    item.close
+    assert_timer @socket, "twirl.op_item_close"
+    assert_timer @socket, "twirl.queue_testing_op_item_close"
+  end
+
+  def test_item_abort
+    @cluster.set "testing", "data"
+    item = @cluster.reserve("testing")
+    item.abort
+    assert_timer @socket, "twirl.op_item_abort"
+    assert_timer @socket, "twirl.queue_testing_op_item_abort"
+  end
+
+  def test_reserve
+    @cluster.set "testing", "data"
+    @cluster.reserve "testing"
+    assert_timer @socket, "twirl.op_reserve"
+    assert_timer @socket, "twirl.queue_testing_op_reserve"
+    assert_counter @socket, "twirl.bytes_op_reserve"
+    assert_counter @socket, "twirl.bytes_queue_testing_op_reserve"
+  end
+
+  def test_peek
+    @cluster.set "testing", "data"
+    @cluster.peek "testing"
+    assert_timer @socket, "twirl.op_peek"
+    assert_timer @socket, "twirl.queue_testing_op_peek"
+    assert_counter @socket, "twirl.bytes_op_peek"
+    assert_counter @socket, "twirl.bytes_queue_testing_op_peek"
+  end
+
+  def test_delete
+    @cluster.delete "testing"
+    assert_timer @socket, "twirl.op_delete"
+    assert_timer @socket, "twirl.queue_testing_op_delete"
+  end
+
+  def test_flush
+    @cluster.flush "testing"
+    assert_timer @socket, "twirl.op_flush"
+    assert_timer @socket, "twirl.queue_testing_op_flush"
+  end
+
+  def test_flush_all
+    @cluster.flush_all
+    assert_timer @socket, "twirl.op_flush_all"
+  end
+
+  def test_version
+    @cluster.version
+    assert_timer @socket, "twirl.op_version"
+  end
+
+  def test_ping
+    @cluster.ping
+    assert_timer @socket, "twirl.op_ping"
+  end
+
+  def test_quit
+    @cluster.quit
+    assert_timer @socket, "twirl.op_quit"
+  end
+
+  def test_stats
+    @cluster.stats
+    assert_timer @socket, "twirl.op_stats"
+  end
+
+  def test_disconnect
+    @cluster.disconnect
+    assert_timer @socket, "twirl.op_disconnect"
+  end
+
+  def test_rotate
+    cluster = build(:cluster, {
+      commands_per_client: 1,
+      instrumenter: ActiveSupport::Notifications,
+    })
+
+    # set data to avoid nil rotation
+    cluster[0].set "testing", "data"
+    cluster[1].set "testing", "data"
+    cluster.get("testing")
+    cluster.get("testing")
+
+    assert_counter @socket, "twirl.op_rotate"
+  end
+
+  def test_get_nil_instruments_rotate
+    @cluster.get("testing")
+    @cluster.get("testing")
+
+    assert_counter @socket, "twirl.op_rotate"
+  end
+
+  def test_get_retries_instrumented
+    @cluster.get("testing")
+    assert_no_timer @socket, "twirl.retries_op_get"
+
+    # force all clients to error
+    @cluster.each do |client|
+      def client.get(*args)
+        raise KJess::NetworkError
+      end
+    end
+
+    begin
+      @cluster.get("testing")
+      flunk "Should raise error and not get here."
+    rescue KJess::NetworkError
+      assert_counter @socket, "twirl.retries_op_get"
+    end
+  end
+
+  def test_set_retries_instrumented
+    @cluster.set("testing", "data")
+    assert_no_timer @socket, "twirl.retries_op_set"
+
+    # force all clients to error
+    @cluster.each do |client|
+      def client.set(*args)
+        raise KJess::NetworkError
+      end
+    end
+
+    begin
+      @cluster.set("testing", "data")
+      flunk "Should raise error and not get here."
+    rescue KJess::NetworkError
+      assert_counter @socket, "twirl.retries_op_set"
+    end
+  end
+end

data/test/instrumenters/memory_test.rb

@@ -0,0 +1,22 @@
+require "helper"
+require "twirl/instrumenters/memory"
+
+class MemoryInstrumenterTest < Minitest::Test
+  def test_initialize
+    instrumentor = Twirl::Instrumenters::Memory.new
+    assert_equal [], instrumentor.events
+  end
+
+  def test_instrument
+    instrumentor = Twirl::Instrumenters::Memory.new
+    name         = 'user.signup'
+    payload      = {:email => 'john@doe.com'}
+    block_result = :yielded
+
+    result = instrumentor.instrument(name, payload) { block_result }
+    assert_equal block_result, result
+
+    event = Twirl::Instrumenters::Memory::Event.new(name, payload, block_result)
+    assert_equal [event], instrumentor.events
+  end
+end

data/test/instrumenters/noop_test.rb

@@ -0,0 +1,16 @@
+require "helper"
+require "twirl/instrumenters/noop"
+
+class NoopInstrumenterTest < Minitest::Test
+  def test_instrument
+    yielded = false
+    Twirl::Instrumenters::Noop.instrument(:foo) { yielded = true }
+    assert yielded
+  end
+
+  def test_instrument_with_payload
+    yielded = false
+    Twirl::Instrumenters::Noop.instrument(:foo, pay: :load) { yielded = true }
+    assert yielded
+  end
+end

data/test/integration/cluster_test.rb

@@ -0,0 +1,199 @@
+require "helper"
+require "twirl/cluster"
+
+class ClusterIntegrationTest < Minitest::Test
+  def test_set
+    cluster = build(:cluster, queues: %w(testing))
+    client = cluster[0]
+    cluster.set("testing", "data")
+    assert_equal "data", client.get("testing")
+  end
+
+  def test_set_rotates
+    cluster = build(:cluster, commands_per_client: 1, queues: %w(testing))
+    client1 = cluster[0]
+    client2 = cluster[1]
+    cluster.set("testing", "data1")
+    cluster.set("testing", "data2")
+    cluster.set("testing", "data3")
+    cluster.set("testing", "data4")
+
+    results = []
+    assert_equal "data1", client1.get("testing")
+    assert_equal "data2", client2.get("testing")
+    assert_equal "data3", client1.get("testing")
+    assert_equal "data4", client2.get("testing")
+  end
+
+  def test_get
+    cluster = build(:cluster, queues: %w(testing))
+    client1 = cluster[0]
+    client2 = cluster[1]
+
+    client1.set("testing", "data1")
+    client1.set("testing", "data2")
+    client2.set("testing", "data3")
+
+    results = []
+    results << cluster.get("testing")
+    results << cluster.get("testing")
+    results << cluster.get("testing")
+    results << cluster.get("testing")
+    results << cluster.get("testing")
+
+    assert_equal ["data1", "data2", nil, "data3", nil], results.map { |result| result.value if result }
+  end
+
+  def test_get_when_item_reserved_raises_error
+    cluster = build(:cluster, queues: %w(testing))
+    client1 = cluster[0]
+
+    client1.set("testing", "data1")
+    cluster.get("testing", open: true)
+
+    assert_raises KJess::ClientError do
+      cluster.get("testing")
+    end
+  end
+
+  def test_reserve_with_item_close_and_item_abort
+    cluster = build(:cluster, queues: %w(testing))
+    client1 = cluster[0]
+    client2 = cluster[1]
+
+    client1.set("testing", "data1")
+    client2.set("testing", "data2")
+
+    item = cluster.reserve("testing")
+    assert_equal "data1", item.value
+
+    item.abort
+    item = cluster.reserve("testing")
+    assert_equal "data1", item.value
+
+    item.close
+    cluster.reserve("testing") # => nil, then rotate
+    item = cluster.reserve("testing")
+    assert_equal "data2", item.value
+
+    cluster.rotate_for_next_op             # force rotation to client1
+    assert_nil cluster.reserve("testing")  # client1 returns nil
+    item.close                             # close item hopefully with client2
+    assert_nil client2.get("testing")      # make sure that client2 queue is empty
+  end
+
+  def test_peek
+    cluster = build(:cluster, queues: %w(testing))
+    client1 = cluster[0]
+    client2 = cluster[1]
+
+    client1.set("testing", "data")
+
+    item = Twirl::Item.new("testing", "data", client1)
+    assert_equal item, cluster.peek("testing")
+    assert_equal "data", client1.get("testing")
+  end
+
+  def test_peek_with_no_items
+    cluster = build(:cluster, queues: %w(testing))
+    assert_nil cluster.peek("testing")
+  end
+
+  def test_version
+    cluster = build(:cluster)
+    expected = {
+      "localhost:9444" => "2.4.1",
+      "localhost:9544" => "2.4.1",
+    }
+
+    assert_equal expected, cluster.version
+  end
+
+  def test_flush
+    cluster = build(:cluster, queues: %w(testing))
+    cluster.each do |client|
+      client.set("testing", "data")
+    end
+    cluster.flush("testing")
+
+    cluster.each do |client|
+      assert_nil client.get("testing")
+    end
+  end
+
+  def test_flush
+    cluster = build(:cluster, queues: %w(testing))
+    cluster.each { |client| client.set("testing", "data") }
+    cluster.flush("testing")
+
+    cluster.each do |client|
+      assert_nil client.get("testing")
+    end
+  end
+
+  def test_flush_all
+    cluster = build(:cluster, queues: %w(testing))
+    cluster.each { |client| client.set("testing", "data") }
+    cluster.flush_all
+
+    cluster.each do |client|
+      assert_nil client.get("testing")
+    end
+  end
+
+  def test_disconnect
+    cluster = build(:cluster)
+    cluster.each(&:ping)
+    expected = [true] * cluster.size
+    assert_equal expected, cluster.map(&:connected?)
+    cluster.disconnect
+    expected = [false] * cluster.size
+    assert_equal expected, cluster.map(&:connected?)
+  end
+
+  def test_quit
+    cluster = build(:cluster)
+    cluster.each(&:ping)
+    expected = [true] * cluster.size
+    assert_equal expected, cluster.map(&:connected?)
+    cluster.quit
+    expected = [false] * cluster.size
+    assert_equal expected, cluster.map(&:connected?)
+  end
+
+  def test_delete
+    cluster = build(:cluster)
+    cluster.each do |client|
+      client.delete("testing")
+      client.set("testing", "data")
+      assert_instance_of Hash, client.queue_stats("testing")
+    end
+    cluster.delete("testing")
+    cluster.each do |client|
+      assert_nil client.queue_stats("testing")
+    end
+  end
+
+  def test_ping
+    cluster = build(:cluster)
+    expected = {
+      "localhost:9444" => true,
+      "localhost:9544" => true,
+    }
+
+    assert_equal expected, cluster.ping
+  end
+
+  def test_stats
+    cluster = build(:cluster)
+    result = cluster.stats
+    assert_equal [
+      "localhost:9444",
+      "localhost:9544",
+    ], result.keys.sort
+
+    result.values.each do |value|
+      assert_instance_of Hash, value
+    end
+  end
+end