plog-ruby 0.0.14

checksums.yaml

@@ -0,0 +1,15 @@
+---
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    NzQ5OTAwNzkxZTRkMmJjMTY0ZGFjMzczOTBmY2RiOGM2YjZjOWQyNw==
+  data.tar.gz: !binary |-
+    ZmE0Y2I1OGEzZmJiMzVjZmYzZDg0MzIxMjY4OTViZjhkMzdlOGNhYQ==
+SHA512:
+  metadata.gz: !binary |-
+    NTkxNGEwZDEyMzhjM2UzYTllZmVmMTkwMmRhOWFjMjk4N2Y0ZTgxNmY0N2Yy
+    MGEwYmMzNWRkYTczOTZlOWRkZWI3MWRlZTljYzliMTlmNTczMmJiMGRlYjkw
+    Y2E0NmExY2M2YmM4NDBkOGI1YmI5OGM0YzgzZTg4NTYwNDcwMTU=
+  data.tar.gz: !binary |-
+    MzRhYjIyMmUyYjY2ZjMzOGVkNzdjODJkOTAxMWNkN2RmZmY3MTE2ZmEyYjNj
+    YmU3YTJiMzdhZGZjNDc0YzA4YTUzZTIzODM0N2Q3Zjk1NGYxNDE1ODA4MjJj
+    NDFmZmFmNTEyMDA4NjgwODZkNjQ4ZGMwM2ExYTcyNWI5ZWQ3ZDM=

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+Gemfile.lock
+.DS_Store

data/Gemfile

@@ -0,0 +1,6 @@
+source "https://rubygems.org"
+gemspec
+
+group :test do
+  gem 'rspec', '~> 2.13.0'
+end

data/README.md

@@ -0,0 +1,27 @@
+# plog
+
+Simple Ruby UDP client for the Plog Kafka forwarder.
+
+### Build & run tests
+
+```
+bundle install
+bundle exec rspec
+```
+
+### Usage
+
+Instantiate a client and start sending messages.
+
+```ruby
+require 'plog'
+client = Plog.new
+client.send("My hovercraft is full of eels.")
+```
+
+You can configure the client at initialization by passing these options:
+
+* `:host` - The host of the Plog process (default: 'localhost')
+* `:port` - The port on which Plog is listening (default: 54321)
+* `:chunk_size` - The maximum payload size for multipart datagrams (default: 64,000)
+* `:logger` - An optional logger instance

data/bin/plogstats

@@ -0,0 +1,23 @@
+#!/usr/bin/env ruby
+
+require 'optparse'
+require 'plog-ruby'
+require 'pp'
+
+options = {:timeout => 3.0}
+
+OptionParser.new do |opts|
+  opts.on '-h', '--host=' do |host|
+    options[:host] = host
+  end
+
+  opts.on '-p', '--port=' do |port|
+    options[:port] = port.to_i
+  end
+
+  opts.on '-t', '--timeout=' do |timeout|
+    options[:timeout] = timeout.to_f
+  end
+end.parse!
+
+pp Plog::Client.new(options).stats(options[:timeout])

data/lib/plog-ruby.rb

@@ -0,0 +1 @@
+require 'plog'

data/lib/plog.rb

@@ -0,0 +1,10 @@
+require 'plog/version'
+require 'plog/packets'
+require 'plog/checksum'
+require 'plog/client'
+
+module Plog
+  def self.new(options={})
+    Plog::Client.new(options)
+  end
+end

data/lib/plog/checksum.rb

@@ -0,0 +1,9 @@
+require 'murmurhash3'
+
+module Plog
+  module Checksum
+    def self.compute(string)
+      MurmurHash3::V32.str_hash(string)
+    end
+  end
+end

data/lib/plog/client.rb

@@ -0,0 +1,161 @@
+require 'json'
+require 'socket'
+require 'thread'
+require 'logger'
+
+module Plog
+  class TimeoutException < StandardError
+  end
+
+  class Client
+    # The protocol version spoken by this client.
+    PROTOCOL_VERSION = Packets::PROTOCOL_VERSION
+    RECV_SIZE = 65_536
+
+    DEFAULT_OPTIONS = {
+      :host => '127.0.0.1',
+      :port => 23456,
+      # Use the socket's default value unless this option is specified.
+      :send_buffer_size => nil,
+      :chunk_size => 64000,
+
+      :large_message_threshold => nil,
+      :on_large_message => nil,
+      :logger => Logger.new(nil)
+    }
+
+    attr_reader :host
+    attr_reader :port
+    attr_reader :send_buffer_size
+    attr_reader :chunk_size
+
+    attr_reader :large_message_threshold
+    attr_reader :on_large_message
+    attr_reader :logger
+
+    attr_reader :last_message_id
+
+    def initialize(options={})
+      options = DEFAULT_OPTIONS.merge(options)
+      @host = options[:host]
+      @port = options[:port]
+      @send_buffer_size = options[:send_buffer_size]
+      @chunk_size = options[:chunk_size]
+      @large_message_threshold = options[:large_message_threshold]
+      @on_large_message = options[:on_large_message]
+      @logger = options[:logger]
+
+      @message_id_mutex = Mutex.new
+      reset_message_id
+    end
+
+    def stats(timeout = 3.0)
+      send_to_socket("\0\0stats")
+      JSON.parse receive_packet_from_socket(timeout)
+    end
+
+    def send(message, options = {})
+      # Interpret the encoding of the string as binary so that chunking occurs
+      # at the byte-level and not at the character-level.
+      message = message.dup.force_encoding('BINARY')
+      notify_large_message(message) if large_message?(message)
+
+      message_id = next_message_id
+      message_length = message.length
+      message_checksum = Checksum.compute(message)
+      chunks = chunk_string(message, chunk_size)
+
+      logger.debug { "Plog: sending (#{message_id}; #{chunks.length} chunk(s))" }
+      chunks.each_with_index do |data, index|
+        send_to_socket(
+          Packets::MultipartMessage.encode(
+            message_id,
+            message_length,
+            message_checksum,
+            chunk_size,
+            chunks.count,
+            index,
+            data,
+            options
+          ))
+      end
+
+      message_id
+    rescue => e
+      logger.error { "Plog: error sending message: #{e}" }
+      raise e
+    end
+
+    def reset
+      reset_message_id
+      close_socket
+    end
+
+    def socket
+      @socket ||= open_socket
+    end
+
+    private
+
+    def large_message?(message)
+      large_message_threshold && message.length > large_message_threshold
+    end
+
+    def notify_large_message(message)
+      on_large_message && on_large_message.call(self, message)
+    end
+
+    def reset_message_id
+      @message_id_mutex.synchronize do
+        @last_message_id = Random.rand(2 ** 32)
+      end
+    end
+
+    def next_message_id
+      @message_id_mutex.synchronize do
+        @last_message_id += 1
+        @last_message_id %= 2 ** 32
+      end
+    end
+
+    def chunk_string(string, size)
+      (0..(string.length - 1) / size).map { |i| string[i * size, size] }
+    end
+
+    def send_to_socket(string)
+      logger.debug { "Plog: writing to socket: #{string.inspect}" }
+      socket.send(string, 0, host, port)
+    rescue => e
+      logger.error { "Plog: error writing to socket: #{e}" }
+      close_socket
+      raise e
+    end
+
+    def receive_packet_from_socket(timeout)
+      logger.debug { "Plog: receiving from socket #{socket} with timeout #{timeout}s" }
+
+      if IO::select([socket], nil, nil, timeout).nil?
+        raise TimeoutException, "No answer in #{timeout}s"
+      end
+
+      socket.recv RECV_SIZE
+    end
+
+    def open_socket
+      socket = UDPSocket.new
+      if send_buffer_size
+        socket.setsockopt(
+          Socket::SOL_SOCKET,
+          Socket::SO_SNDBUF,
+          send_buffer_size
+        )
+      end
+      socket
+    end
+
+    def close_socket
+      @socket.close rescue nil
+      @socket = nil
+    end
+  end
+end

data/lib/plog/packets.rb

@@ -0,0 +1,9 @@
+require 'plog/packets/multipart_message'
+
+module Plog
+  module Packets
+    PROTOCOL_VERSION = 0
+    # Identifies the content of each packet.
+    TYPE_MULTIPART_MESSAGE = 1
+  end
+end

data/lib/plog/packets/multipart_message.rb

@@ -0,0 +1,42 @@
+module Plog
+  module Packets
+
+    module MultipartMessage
+      def self.encode(message_id, length, checksum, chunk_size, count, index, payload, options = {})
+        message = [
+            PROTOCOL_VERSION,
+            TYPE_MULTIPART_MESSAGE,
+            count,
+            index,
+            chunk_size,
+            message_id,
+            length,
+            checksum]
+
+        # Plog encoding: https://github.com/airbnb/plog
+        template = 'CCS>S>S>L>l>L>S>x2'
+
+        # Generate pack template for tags
+        tags = options[:tags]
+        if tags.nil? || tags.empty?
+          message << 0
+        else
+          tag_len = 0
+          tags.each do |tag|
+            len = tag.length + 1 # extra byte for '\0'
+            template += "a#{len}"
+            tag_len += len
+          end
+          message << tag_len
+          message.concat(tags)
+        end
+
+        message << payload
+        template += 'a*'
+
+        return message.pack(template)
+      end
+    end
+
+  end
+end

data/lib/plog/version.rb

@@ -0,0 +1,3 @@
+module Plog
+  VERSION = '0.0.14'
+end

data/plog-ruby.gemspec

@@ -0,0 +1,22 @@
+# encoding: utf-8
+$:.push File.expand_path("../lib", __FILE__)
+
+require 'plog/version'
+
+Gem::Specification.new do |s|
+  s.name         = "plog-ruby"
+  s.version      = Plog::VERSION
+  s.platform     = Gem::Platform::RUBY
+  s.authors      = ["Nelson Gauthier"]
+  s.email        = ["nelson@airbnb.com"]
+  s.homepage     = "https://github.com/airbnb/plog-ruby"
+  s.summary      = "Ruby client for Plog."
+  s.description  = "Send messages via UDP to the Plog Kafka forwarder."
+
+  s.files        = `git ls-files`.split("\n")
+  s.test_files   = `git ls-files -- spec/*`.split("\n")
+  s.require_path = 'lib'
+  s.executables  = 'plogstats'
+
+  s.add_runtime_dependency 'murmurhash3', '~> 0.1'
+end

data/spec/lib/plog/client_spec.rb

@@ -0,0 +1,271 @@
+require 'spec_helper'
+
+describe Plog::Client do
+
+  let(:chunk_size) { 5 }
+  let(:client_options) { { :chunk_size => chunk_size } }
+  subject { Plog::Client.new(client_options) }
+
+  let(:udp_socket) do
+    double(UDPSocket).tap do |udp_socket|
+      udp_socket.stub(:send)
+      udp_socket.stub(:close)
+      udp_socket.stub(:recv)
+    end
+  end
+
+  before do
+    UDPSocket.stub(:new).and_return(udp_socket)
+  end
+
+  describe '#stats' do
+    let(:select_value) { [[udp_socket], [], []] }
+    let(:recv_value) { '{"foo":1}' }
+
+    before do
+      IO.stub(:select).and_return(select_value)
+      udp_socket.stub(:recv) { recv_value }
+    end
+
+    context 'timing out' do
+      let(:select_value) { nil }
+      it 'raises a TimeoutException' do
+        expect {subject.stats}.to raise_error Plog::TimeoutException
+      end
+    end
+
+    it 'sends a stats request' do
+      udp_socket.should_receive(:send) do |req, *extraneous|
+        expect(req.downcase).to start_with("\0\0stat")
+      end
+
+      subject.stats
+    end
+
+    it 'returns a statistics object from deserializing JSON' do
+      expect(subject.stats).to eq({'foo' => 1})
+    end
+  end
+
+  describe '#send' do
+    let(:message)  { 'xxx' }
+    let(:checksum) { 200 }
+
+    before do
+      Plog::Checksum.stub(:compute).and_return(checksum)
+    end
+
+    it "constructs a UDP socket" do
+      UDPSocket.should_receive(:new).and_return(udp_socket)
+      subject.send(message)
+    end
+
+    context "when a send buffer size is specified" do
+      before do
+        client_options.merge!(:send_buffer_size => 1000)
+      end
+
+      it "sets the SO_SNDBUF socket option" do
+        udp_socket.should_receive(:setsockopt).with(
+          Socket::SOL_SOCKET,
+          Socket::SO_SNDBUF,
+          1000)
+        subject.send(message)
+      end
+    end
+
+    it "contacts the given host and port" do
+      udp_socket.should_receive(:send).with(anything(), 0, subject.host, subject.port)
+      subject.send(message)
+    end
+
+    it "encodes the message id, message length and chunk size" do
+      first_id = subject.last_message_id
+      Plog::Packets::MultipartMessage.should_receive(:encode).with(
+        first_id + 1,
+        message.length,
+        checksum,
+        chunk_size,
+        anything(),
+        anything(),
+        message,
+        {}
+      ).and_call_original
+      subject.send(message)
+    end
+
+    it "returns an monotonically increasing message id" do
+      first_id = subject.last_message_id
+      expect(subject.send(message)).to eq(first_id + 1)
+      expect(subject.send(message)).to eq(first_id + 2)
+    end
+
+    it "reuses the same socket" do
+      UDPSocket.should_receive(:new).once.and_return(udp_socket)
+      2.times { subject.send(message) }
+    end
+
+    describe 'large messages' do
+      let(:large_message_threshold) { nil }
+      let(:callback) { lambda { |c, m| } }
+
+      before do
+        client_options.merge!({
+          :large_message_threshold => large_message_threshold,
+          :on_large_message => callback
+        })
+      end
+
+      context "when the large message threshold is nil" do
+        let(:large_message_threshold) { nil }
+
+        it "does not invoke the callback" do
+          callback.should_not_receive(:call)
+          subject.send(message)
+        end
+      end
+
+      context "when the large message threshold is given" do
+        let(:large_message_threshold) { 2 }
+        let(:message)  { 'xxx' }
+
+        it "invokes the callback with the client and message" do
+          callback.should_receive(:call).with(subject, message)
+          subject.send(message)
+        end
+
+        context "when the callback is nil" do
+          let(:callback) { nil }
+
+          it "doesn't raise" do
+            subject.send(message)
+          end
+        end
+      end
+    end
+
+    describe 'message id' do
+      before do
+        @message_ids = []
+        Plog::Packets::MultipartMessage.stub(:encode) do |message_id, _, _, _, _, _, _|
+          @message_ids << message_id
+        end
+      end
+
+      it "encodes each message with a monotonically increasing message id" do
+        first_id = subject.last_message_id
+        expected_sequence = (first_id + 1...first_id + 6).to_a
+        5.times { subject.send(message) }
+        expect(@message_ids).to eq(expected_sequence)
+      end
+    end
+
+    describe 'chunking' do
+      let(:chunk_size) { 5 }
+      let(:message) { 'AAAA' }
+      let(:expected_chunks) { ['AAAA'] }
+
+      before do
+        @sent_datagrams = []
+        Plog::Packets::MultipartMessage.stub(:encode) do |_, _, _, _, count, index, data|
+          [count, index, data]
+        end
+        udp_socket.stub(:send) do |datagram, _, _, _|
+          @sent_datagrams << datagram
+        end
+      end
+
+      def validate_datagrams
+        # Reassemble the message as binary and verify the counts and indexes.
+        reassembled_message = "".force_encoding('BINARY')
+        @sent_datagrams.each_with_index do |(count, index, data), datagram_index|
+          expect(count.to_i).to eq(expected_chunks.count)
+          expect(index.to_i).to eq(datagram_index)
+          expect(data).to eq(expected_chunks[datagram_index].force_encoding('BINARY'))
+          reassembled_message += data
+        end
+        # Convert the message back to the original encoding and verify.
+        reassembled_message.force_encoding(message.encoding)
+        expect(reassembled_message).to eq(message)
+      end
+
+      context "when the message length is lower than the chunk size" do
+        let(:chunk_size) { 5 }
+        let(:message) { "A" * (chunk_size - 1) }
+        let(:expected_chunks)  { [message] }
+
+        it "encodes the message and sends it as a single packet" do
+          subject.send(message)
+          validate_datagrams
+        end
+      end
+
+      context "when the message is large than the chunk size" do
+        let(:chunk_size) { 5 }
+        let(:message) { "A" * (chunk_size + 1) }
+        let(:expected_chunks)  { ["A" * chunk_size, "A"] }
+
+        it "chunks the message and sends it as many packets" do
+          subject.send(message)
+          validate_datagrams
+        end
+      end
+
+      context "when the message contains multi-byte encoded characters" do
+        let(:chunk_size) { 5 }
+        let(:message) { "\u00E9ABCDEFGH" }
+        let(:expected_chunks) { [
+            "\u00E9ABC",
+            "DEFGH"
+          ]}
+
+        it "correctly chunks the message" do
+          subject.send(message)
+          validate_datagrams
+        end
+      end
+    end
+
+    describe 'exceptions' do
+
+      context "when the socket operation raises" do
+        it "closes and re-opens the socket" do
+          udp_socket.stub(:send).and_raise
+          udp_socket.should_receive(:close).once
+          expect { subject.send(message) }.to raise_error
+
+          udp_socket.stub(:send) {}
+          UDPSocket.should_receive(:new).once.and_return(udp_socket)
+          subject.send(message)
+        end
+      end
+
+    end
+
+  end
+
+  describe '#reset' do
+    let(:message)  { 'xxx' }
+
+    it "chooses a new random message id" do
+      Random.stub(:rand).and_return(2)
+      subject.send(message)
+      expect(subject.last_message_id).to eq(3)
+
+      Random.stub(:rand).and_return(5)
+      subject.reset
+      expect(subject.last_message_id).to eq(5)
+    end
+
+    context "with an initialized socket" do
+      before do
+        subject.send(message)
+        subject.socket.should_receive(:close)
+        subject.reset
+      end
+
+    end
+
+  end
+
+end

data/spec/lib/plog/packets/multipart_message_spec.rb

@@ -0,0 +1,100 @@
+require 'spec_helper'
+
+describe Plog::Packets::MultipartMessage do
+
+  describe '.encode' do
+
+    # Each of these values were chosen to fit in a single byte.
+    let(:message_id) { 1 }
+    let(:length)     { 2 }
+    let(:checksum)   { 3 }
+    let(:chunk_size) { 4 }
+    let(:count)      { 5 }
+    let(:index)      { 6 }
+    let(:payload)    { 'xxx' }
+
+    let(:encoded) do
+      Plog::Packets::MultipartMessage.encode(
+        message_id,
+        length,
+        checksum,
+        chunk_size,
+        count,
+        index,
+        payload
+      )
+    end
+
+    def encoded_range(first, last)
+      encoded[first..last].bytes.to_a
+    end
+
+    it "encodes a string with length 24 + payload length" do
+      expect(encoded.length).to eq(24 + payload.length)
+    end
+
+    it "encodes the protocol version as the first byte" do
+      expect(encoded_range(0, 0)).to eq([Plog::Client::PROTOCOL_VERSION])
+    end
+
+    it "encodes the command as the second byte" do
+      expect(encoded_range(1, 1)).to eq([Plog::Packets::TYPE_MULTIPART_MESSAGE])
+    end
+
+    it "encodes the multipart packet count big endian as bytes 02-03" do
+      expect(encoded_range(2, 3)).to eq([0, count])
+    end
+
+    it "encodes the multipart packet index big endian as bytes 04-05" do
+      expect(encoded_range(4, 5)).to eq([0, index])
+    end
+
+    it "encodes the chunk size big endian as bytes 06-07" do
+      expect(encoded_range(6, 7)).to eq([0, chunk_size])
+    end
+
+    it "encodes the message id big endian as bytes 08-11" do
+      expect(encoded_range(8, 11)).to eq([0, 0, 0, message_id])
+    end
+
+    it "encodes the total message length as bytes 12-15" do
+      expect(encoded_range(12, 15)).to eq([0, 0, 0, length])
+    end
+
+    it "encodes the message checksum as bytes 16-19" do
+      expect(encoded_range(16, 19)).to eq([0, 0, 0, checksum])
+    end
+
+    it "encodes zero padding for the reserved segment as bytes 20-23" do
+      expect(encoded_range(20, 23)).to eq([0, 0, 0, 0])
+    end
+
+    let(:tags)       { ['tag1', 'tag2'] }
+    let(:encoded_with_tags) do
+      Plog::Packets::MultipartMessage.encode(
+        message_id,
+        length,
+        checksum,
+        chunk_size,
+        count,
+        index,
+        payload,
+        tags: tags
+      )
+    end
+
+    def encoded_with_tags_range(first, last)
+      encoded_with_tags[first..last].bytes.to_a
+    end
+
+    it "encodes the tags length as bytes 20-21" do
+      expect(encoded_with_tags_range(20, 21)).to eq([0, 10])
+    end
+
+    it "encodes a tagged string with length 24 + tag length + payload length" do
+      expect(encoded_with_tags.length).to eq(24 + 10 + payload.length)
+    end
+
+  end
+
+end

data/spec/lib/plog_spec.rb

@@ -0,0 +1,18 @@
+require 'spec_helper'
+
+describe Plog do
+
+  describe '.new' do
+    let(:options) { { :foo => :bar } }
+
+    it "returns a plog client" do
+      expect(Plog.new).to be_an_instance_of(Plog::Client)
+    end
+
+    it "passes options to the client initializer" do
+      Plog::Client.should_receive(:new).with(options).and_call_original
+      Plog.new(options)
+    end
+  end
+
+end

data/spec/spec_helper.rb

@@ -0,0 +1,5 @@
+require 'plog'
+
+RSpec.configure do |config|
+  config.color_enabled = true
+end

metadata

@@ -0,0 +1,78 @@
+--- !ruby/object:Gem::Specification
+name: plog-ruby
+version: !ruby/object:Gem::Version
+  version: 0.0.14
+platform: ruby
+authors:
+- Nelson Gauthier
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2015-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: murmurhash3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '0.1'
+description: Send messages via UDP to the Plog Kafka forwarder.
+email:
+- nelson@airbnb.com
+executables:
+- plogstats
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- README.md
+- bin/plogstats
+- lib/plog-ruby.rb
+- lib/plog.rb
+- lib/plog/checksum.rb
+- lib/plog/client.rb
+- lib/plog/packets.rb
+- lib/plog/packets/multipart_message.rb
+- lib/plog/version.rb
+- plog-ruby.gemspec
+- spec/lib/plog/client_spec.rb
+- spec/lib/plog/packets/multipart_message_spec.rb
+- spec/lib/plog_spec.rb
+- spec/spec_helper.rb
+homepage: https://github.com/airbnb/plog-ruby
+licenses: []
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.4.1
+signing_key: 
+specification_version: 4
+summary: Ruby client for Plog.
+test_files:
+- spec/lib/plog/client_spec.rb
+- spec/lib/plog/packets/multipart_message_spec.rb
+- spec/lib/plog_spec.rb
+- spec/spec_helper.rb