ruby_nsq 0.0.1

data/History.md

@@ -0,0 +1,7 @@
+Changelog
+=====================
+
+0.0.1
+-----
+
+ - Initial release

data/LICENSE.txt

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2012 Clarity Services, Inc.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -0,0 +1,71 @@
+# ruby_nsq
+
+https://github.com/ClarityServices/ruby_nsq
+
+## Description:
+
+Ruby client for the [NSQ](https://github.com/bitly/nsq) realtime message processing system.
+
+## Install:
+
+  gem install ruby_nsq
+
+## Usage:
+
+See [examples](https://github.com/ClarityServices/ruby_nsq/tree/master/examples)
+
+Simple example for synchronous message handling:
+```
+require 'nsq'
+
+reader = NSQ.create_reader(:nsqd_tcp_addresses => '127.0.0.1:4150')
+# Subscribe to topic=test channel=simple
+reader.subscribe('test', 'simple') do |message|
+  # If this block raises an exception, then the message will be requeued.
+  puts "Read #{message.body}"
+end
+reader.run   # Doesn't return until reader.stop is called
+puts 'Reader stopped'
+```
+
+Advanced example demonstrating asynchronous handling of messages on multiple threads:
+```
+require 'nsq'
+
+foo_worker_count = 50
+bar_worker_count = 30
+baz_worker_count = 20
+
+reader = NSQ.create_reader(:nsqd_tcp_addresses => '127.0.0.1:4150')
+
+foo_subscriber  = reader.subscribe('test',  'foo', :max_in_flight => foo_worker_count)
+bar_subscriber  = reader.subscribe('test2', 'bar', :max_in_flight => bar_worker_count)
+baz_subscriber  = reader.subscribe('test2', 'baz', :max_in_flight => baz_worker_count)
+
+foo_threads = foo_worker_count.times.map do |i|
+  Thread.new(i) do |i|
+    foo_subscriber.run do |message|
+      puts 'Foo[%02d] read: %s' % i, message.body
+      sleep rand(10)  # Dummy processing of message
+    end
+    puts 'Foo[%02d] thread exiting' % i
+  end
+end
+
+bar_threads = ... Same kind of thing as above ...
+baz_threads = ... Same kind of thing as above ...
+
+reader.run   # Doesn't return until reader.stop is called
+puts 'Reader stopped'
+foo_threads.each(&:join)
+bar_threads.each(&:join)
+baz_threads.each(&:join)
+```
+
+## TODO:
+
+* Fix timestamp
+
+* Implement lookupd
+
+* Tests!

data/Rakefile

@@ -0,0 +1,34 @@
+#!/usr/bin/env rake
+begin
+  require 'bundler/setup'
+rescue LoadError
+  puts 'You must `gem install bundler` and `bundle install` to run rake tasks'
+end
+begin
+  require 'rdoc/task'
+rescue LoadError
+  require 'rdoc/rdoc'
+  require 'rake/rdoctask'
+  RDoc::Task = Rake::RDocTask
+end
+
+RDoc::Task.new(:rdoc) do |rdoc|
+  rdoc.rdoc_dir = 'rdoc'
+  rdoc.title    = 'ruby_nsq'
+  rdoc.options << '--line-numbers'
+  rdoc.rdoc_files.include('README.md')
+  rdoc.rdoc_files.include('lib/**/*.rb')
+end
+
+Bundler::GemHelper.install_tasks
+
+require 'rake/testtask'
+
+Rake::TestTask.new(:test) do |t|
+  t.libs << 'lib'
+  t.libs << 'test'
+  t.pattern = 'test/**/*_test.rb'
+  t.verbose = false
+end
+
+task :default => :test

data/examples/async/reader.rb

@@ -0,0 +1,58 @@
+#!/usr/bin/env ruby
+
+require 'nsq'
+require 'thread'
+require 'logger'
+
+x_worker_count = 50
+y_worker_count = 30
+z_worker_count = 20
+
+puts 'Press enter to start and enter to finish'
+$stdin.gets
+
+reader = NSQ.create_reader(
+    :nsqd_tcp_addresses => '127.0.0.1:4150',
+    :logger_level       => Logger::DEBUG
+)
+
+x_subscriber   = reader.subscribe('test_xy', 'x', :max_in_flight => x_worker_count)
+y_subscriber   = reader.subscribe('test_xy', 'y', :max_in_flight => y_worker_count)
+z_subscriber   = reader.subscribe('test_z',  'z', :max_in_flight => z_worker_count)
+
+class MyThread < Thread
+  attr_accessor :message_count
+  def initialize(index, subscriber, char)
+    @index = index
+    super do |i, subscriber, char|
+      @message_count = 0
+      subscriber.run do |message|
+        eval message.body
+        print char
+        @message_count += 1
+      end
+      print char.upcase
+    end
+  end
+end
+
+threads = {}
+[[x_subscriber, x_worker_count, 'x'], [y_subscriber, y_worker_count, 'y'], [z_subscriber, z_worker_count, 'z']].each do |subscriber, count, char|
+  threads[char] = count.times.map do |i|
+    MyThread.new(i, subscriber, char)
+  end
+end
+
+main_thread = Thread.new do
+  reader.run
+end
+$stdin.gets
+puts 'Exiting...'
+reader.stop
+main_thread.join
+threads.each_value { |arr| arr.each(&:join) }
+puts
+puts "Summary of worker message counts"
+threads.each do |char, arr|
+  puts "#{char} -  #{arr.map(&:message_count).join(' ')}"
+end

data/examples/async/writer.rb

@@ -0,0 +1,17 @@
+#!/usr/bin/env ruby
+
+if ARGV.length != 3
+  $stderr.puts "ruby writer.rb <topic> <count> <eval-string>"
+  $stderr.puts "  where <topic> is either test_xy or test_z"
+  $stderr.puts "  and <eval-string> could be something like 'sleep rand(100)/10.0'"
+  $stderr.puts "  Example: ./writer.rb test_xy 500 'sleep rand(100)/10.0'"
+  $stderr.puts "       or: ./writer.rb test_z 5000 nil"
+  exit 1
+end
+topic       = ARGV[0]
+count       = ARGV[1].to_i
+eval_string = ARGV[2]
+# TODO: Figure out TCP protocol
+count.times do
+  system "curl -d #{eval_string.inspect} 'http://127.0.0.1:4151/put?topic=#{topic}'"
+end

data/examples/simple/reader.rb

@@ -0,0 +1,25 @@
+#!/usr/bin/env ruby
+
+require 'nsq'
+require 'logger'
+
+puts 'Press enter to start and enter to finish'
+gets
+reader = NSQ.create_reader(
+    :nsqd_tcp_addresses => '127.0.0.1:4150',
+    #:logger_level       => Logger::DEBUG
+)
+thread = Thread.new do
+  begin
+    reader.subscribe('test', 'simple') do |message|
+      puts "Read #{message.body}"
+    end
+    reader.run
+  rescue Exception => e
+    $stderr.puts "Unexpected error: #{e.message}\n\t#{e.backtrace.join("\n\t")}"
+  end
+  puts 'Reader exiting'
+end
+gets
+reader.stop
+thread.join

data/lib/nsq/backoff_timer.rb

@@ -0,0 +1,42 @@
+# Stolen from pynsq library since somebodies thought about this a lot more than me
+module NSQ
+  # This is a timer that is smart about backing off exponentially when there are problems
+  class BackoffTimer
+
+    attr_reader :min_interval, :max_interval, :short_interval, :long_interval
+
+    def initialize(min_interval, max_interval, ratio=0.25, short_length=10, long_length=250)
+      @min_interval    = min_interval.to_f
+      @max_interval    = max_interval.to_f
+      ratio            = ratio.to_f
+
+      @max_short_timer = (@max_interval - @min_interval) * ratio
+      @max_long_timer  = (@max_interval - @min_interval) * (1.0 - ratio)
+      @short_unit      = @max_short_timer / short_length
+      @long_unit       = @max_long_timer / long_length
+
+      @short_interval  = 0.0
+      @long_interval   = 0.0
+    end
+
+    # Update the timer to reflect a successful call
+    def success
+      @short_interval -= @short_unit
+      @long_interval  -= @long_unit
+      @short_interval  = [@short_interval, 0.0].max
+      @long_interval   = [@long_interval, 0.0].max
+    end
+
+    # Update the timer to reflect a failed call
+    def failure
+      @short_interval += @short_unit
+      @long_interval  += @long_unit
+      @short_interval  = [@short_interval, @max_short_timer].min
+      @long_interval   = [@long_interval, @max_long_timer].min
+    end
+    
+    def interval
+      @min_interval + @short_interval + @long_interval
+    end
+  end
+end

data/lib/nsq/connection.rb

@@ -0,0 +1,222 @@
+require 'monitor'
+require 'thread'  #Mutex
+
+module NSQ
+  class Connection
+    attr_reader :name
+
+    def initialize(reader, subscriber, host, port)
+      @reader        = reader
+      @subscriber    = subscriber
+      @selector      = reader.selector
+      @host          = host
+      @port          = port
+      @name          = "#{subscriber.name}:#{host}:#{port}"
+      @write_monitor = Monitor.new
+      @ready_mutex   = Mutex.new
+      @sending_ready = false
+
+      # Connect states :init, :interval, :connecting, :connected, :closed
+      @connect_state = :init
+
+      @next_connection_time     = nil
+      @next_ready_time          = nil
+      @connection_backoff_timer = nil
+      @ready_backoff_timer      = @subscriber.create_ready_backoff_timer
+
+      connect
+    end
+
+    def send_init(topic, channel, short_id, long_id)
+      write NSQ::MAGIC_V2
+      write "SUB #{topic} #{channel} #{short_id} #{long_id}\n"
+      self.send_ready
+    end
+
+    def send_ready
+      @ready_count = @subscriber.ready_count
+      write "RDY #{@ready_count}\n" unless @subscriber.stopped?
+      @sending_ready = false
+    end
+
+    def send_finish(id, success)
+      write "FIN #{id}\n"
+      @ready_mutex.synchronize do
+        @ready_count -= 1
+        if success
+          @ready_backoff_timer.success
+        else
+          @ready_backoff_timer.failure
+        end
+        check_ready
+      end
+    end
+
+    def send_requeue(id, time_ms)
+      write "REQ #{id} #{time_ms}\n"
+      @ready_mutex.synchronize do
+        @ready_count -= 1
+        @ready_backoff_timer.failure
+        check_ready
+      end
+    end
+
+    def reset
+      return unless verify_connect_state?(:connecting, :connected)
+      # Close with the hopes of re-establishing
+      close(false)
+      @write_monitor.synchronize do
+        return unless verify_connect_state?(:init)
+        @connection_backoff_timer ||= @subscriber.create_connection_backoff_timer
+        @connection_backoff_timer.failure
+        interval = @connection_backoff_timer.interval
+        if interval > 0
+          @connect_state = :interval
+          NSQ.logger.debug {"#{self}: Reattempting connection in #{interval} seconds"}
+          @reader.add_timeout(interval) do
+            connect
+          end
+        else
+          connect
+        end
+      end
+    end
+
+    def close(permanent=true)
+      NSQ.logger.debug {"#{@name}: Closing..."}
+      @write_monitor.synchronize do
+        begin
+          @selector.deregister(@socket)
+          # Use straight socket to write otherwise we need to use Monitor instead of Mutex
+          @socket.write "CLS\n"
+          @socket.close
+        rescue Exception => e
+        ensure
+          @connect_state = permanent ? :closed : :init
+          @socket        = nil
+        end
+      end
+    end
+
+    def connect
+      return unless verify_connect_state?(:init, :interval)
+      NSQ.logger.debug {"#{self}: Beginning connect"}
+      @connect_state = :connecting
+      @buffer        = ''
+      @connecting    = false
+      @connected     = false
+      @ready_count   = 0
+      @socket        = Socket.new(Socket::AF_INET, Socket::SOCK_STREAM, 0)
+      @sockaddr      = Socket.pack_sockaddr_in(@port, @host)
+      @monitor       = @selector.register(@socket, :w)
+      @monitor.value = proc { do_connect }
+      do_connect
+    end
+
+    private
+
+    def do_connect
+      @write_monitor.synchronize do
+        return unless verify_connect_state?(:connecting)
+        begin
+          @socket.connect_nonblock(@sockaddr)
+          # Apparently we always throw an exception here
+          NSQ.logger.debug {"#{self}: do_connect fell thru without throwing an exception"}
+        rescue Errno::EINPROGRESS
+          NSQ.logger.debug {"#{self}: do_connect - connect in progress"}
+        rescue Errno::EISCONN
+          NSQ.logger.debug {"#{self}: do_connect - connection complete"}
+          @selector.deregister(@socket)
+          monitor = @selector.register(@socket, :r)
+          monitor.value = proc { read_messages }
+          @connect_state = :connected
+          # The assumption for connections is that a good connection means the server is good, no ramping back up like ready counts
+          @connection_backoff_timer = nil
+          @subscriber.handle_connection(self)
+        rescue SystemCallError => e
+          @subscriber.handle_io_error(self, e)
+        end
+      end
+    end
+
+    def check_ready
+      if !@sending_ready && @ready_count <= @subscriber.ready_threshold
+        interval = @ready_backoff_timer.interval
+        if interval == 0.0
+          send_ready
+        else
+          NSQ.logger.debug {"#{self}: Delaying READY for #{interval} seconds"}
+          @sending_ready = true
+          @reader.add_timeout(interval) do
+            send_ready
+          end
+        end
+      end
+    end
+
+    def read_messages
+      @buffer << @socket.read_nonblock(4096)
+      while @buffer.length >= 8
+        size, frame = @buffer.unpack('NN')
+        break if @buffer.length < 4+size
+        case frame
+          when NSQ::FRAME_TYPE_RESPONSE
+            if @buffer[8,11] == "_heartbeat_"
+              send_nop
+              @subscriber.handle_heartbeat(self)
+              @buffer = @buffer[(4+size)..-1]
+            else
+              NSQ.logger.error("I don't know what to do with the rest of this buffer: #{@buffer[8,size-4].inspect}") if @buffer.length > 8
+              @buffer = @buffer[(4+size)..-1]
+            end
+          when NSQ::FRAME_TYPE_ERROR
+            @subscriber.handle_frame_error(self, @buffer[8, size-4])
+            @buffer = @buffer[(4+size)..-1]
+          when NSQ::FRAME_TYPE_MESSAGE
+            raise "Bad message: #{@buffer.inspect}" if size < 30
+            ts_hi, ts_lo, attempts, id = @buffer.unpack('@8NNna16')
+            body = @buffer[34, size-30]
+            message = Message.new(self, id, ts_hi, ts_lo, attempts, body)
+            @buffer = @buffer[(4+size)..-1]
+            NSQ.logger.debug {"#{self}: Read message=#{message}"}
+            @subscriber.handle_message(self, message)
+          else
+            raise "Unrecognized message frame: #{frame} buffer=#{@buffer.inspect}"
+        end
+      end
+    rescue Exception => e
+      @subscriber.handle_io_error(self, e)
+    end
+
+    def send_nop
+      write "NOP\n"
+    end
+
+    def write(msg)
+      NSQ.logger.debug {"#{@name}: Sending #{msg.inspect}"}
+      # We should only ever have one reader but we can have multiple writers
+      @write_monitor.synchronize do
+        @socket.write(msg) if verify_connect_state?(:connected)
+      end
+    end
+
+    def to_s
+      @name
+    end
+
+    private
+
+    def verify_connect_state?(*states)
+      return true if states.include?(@connect_state)
+      NSQ.logger.error("Unexpected connect state of #{@connect_state}, expected to be in #{states.inspect}\n\t#{caller[0]}")
+      if @connect_state != :closed
+        # Likely in a bug state.
+        # I don't want to get in an endless loop of exceptions.  Is this a good idea or bad?  Maybe close to deregister first
+        # Attempt recovery
+        @connect_state = :init
+        connect
+      end
+      return false
+    end
+  end
+end

data/lib/nsq/loggable.rb

@@ -0,0 +1,23 @@
+module NSQ
+  module Loggable
+    def logger
+      @logger ||= (rails_logger || default_logger)
+    end
+
+    def rails_logger
+      (defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger) ||
+      (defined?(RAILS_DEFAULT_LOGGER) && RAILS_DEFAULT_LOGGER.respond_to?(:debug) && RAILS_DEFAULT_LOGGER)
+    end
+
+    def default_logger
+      require 'logger'
+      l = Logger.new($stdout)
+      l.level = Logger::INFO
+      l
+    end
+
+    def logger=(logger)
+      @logger = logger
+    end
+  end
+end

data/lib/nsq/message.rb

@@ -0,0 +1,22 @@
+module NSQ
+  class Message
+    attr_reader :connection, :id, :attempts, :body
+
+    def initialize(connection, id, timestamp_high, timestamp_low, attempts, body)
+      @connection     = connection
+      @id             = id
+      @timestamp_high = timestamp_high
+      @timestamp_low  = timestamp_low
+      @attempts       = attempts
+      @body           = body
+    end
+
+    def timestamp
+      Time.at((@timestamp_high * 2**32 + @timestamp_low) / 1000000000.0)
+    end
+
+    def to_s
+      "#{connection} id=#{id} timestamp=#{timestamp} attempts=#{attempts} body=#{body}"
+    end
+  end
+end

data/lib/nsq/queue_subscriber.rb

@@ -0,0 +1,45 @@
+require 'thread' #Mutex
+
+module NSQ
+  class QueueSubscriber < Subscriber
+    def initialize(reader, topic, channel, options)
+      super
+      @queue     = Queue.new
+      @run_mutex = Mutex.new
+      @run_count = 0
+    end
+
+    def ready_count
+      # Return the minimum of Subscriber#ready_count and the amount of space left in the queue
+      [super, self.max_in_flight - @queue.size].min
+    end
+
+    def handle_message(connection, message)
+      @queue << [connection, message]
+    end
+
+    def run(&block)
+      @run_mutex.synchronize { @run_count += 1}
+      until @stopped
+        pair = @queue.pop
+        if pair == :stop
+          @queue << :stop
+          return
+        end
+        connection, message = pair
+        process_message(connection, message, &block)
+      end
+    ensure
+      @run_mutex.synchronize { @run_count -= 1}
+    end
+
+    def stop
+      @stopped = true
+      # Give the threads something to pop
+      @queue << :stop
+      # TODO: Put a max time on this so we don't potentially hang
+      sleep 1 while @run_count > 0
+      super
+    end
+  end
+end

data/lib/nsq/reader.rb

@@ -0,0 +1,99 @@
+require 'socket'
+require 'thread'
+require 'monitor'
+require 'nio'
+#require 'thread_safe'
+
+module NSQ
+  class Reader
+    attr_reader :name, :long_id, :short_id, :selector, :options
+
+    def initialize(options={})
+      @options                = options
+      @nsqd_tcp_addresses     = s_to_a(options[:nsqd_tcp_addresses])
+      @lookupd_http_addresses = s_to_a(options[:lookupd_http_addresses])
+      @lookupd_poll_interval  = options[:lookupd_poll_interval]        || 120
+      @long_id                = options[:long_id]                      || Socket.gethostname
+      @short_id               = options[:short_id]                     || @long_id.split('.')[0]
+      NSQ.logger              = options[:logger] if options[:logger]
+      NSQ.logger.level        = options[:logger_level] if options[:logger_level]
+
+      @selector               = ::NIO::Selector.new
+      @timer                  = Timer.new(@selector)
+      @topic_count            = Hash.new(0)
+      @subscribers            = {}
+      @subscriber_mutex       = Monitor.new
+      @name                   = "#{@long_id}:#{@short_id}"
+
+      raise 'Must pass either option :nsqd_tcp_addresses or :lookupd_http_addresses' if @nsqd_tcp_addresses.empty? && @lookupd_http_addresses.empty?
+
+      # TODO: If the messages are failing, the backoff timer will exponentially increase a timeout before sending a RDY
+      #self.backoff_timer = dict((k, BackoffTimer.BackoffTimer(0, 120)) for k in self.task_lookup.keys())
+        
+      @conns = {}
+      @last_lookup = nil
+
+      @logger.info("starting reader for topic '%s'..." % self.topic) if @logger
+    end
+
+    def subscribe(topic, channel, subscribe_options={}, &block)
+      NSQ.assert_topic_and_channel_valid(topic, channel)
+      @topic     = topic
+      @channel   = channel
+      subscriber = nil
+      name       = "#{topic}:#{channel}"
+      @subscriber_mutex.synchronize do
+        raise "Already subscribed to #{name}" if @subscribers[name]
+        subscriber_class = block_given? ? Subscriber : QueueSubscriber
+        subscriber = @subscribers[name] = subscriber_class.new(self, topic, channel, subscribe_options, &block)
+      end
+
+      @nsqd_tcp_addresses.each do |addr|
+        address, port = addr.split(':')
+        subscriber.add_connection(address, port.to_i)
+      end
+      subscriber
+    end
+
+    def unsubscribe(topic, channel)
+      name = "#{topic}:#{channel}"
+      @subscriber_mutex.synchronize do
+        subscriber = @subscribers[name]
+        return unless subscriber
+        subscriber.stop
+        @subscribers.delete(name)
+      end
+    end
+
+    def run
+      @stopped = false
+      until @stopped do
+        if (Time.now.to_i - @last_lookup.to_i) > @lookupd_poll_interval
+          # Do lookupd
+        end
+        @selector.select(@timer.next_interval) { |m| m.value.call }
+      end
+    end
+
+    def stop
+      NSQ.logger.info("#{self}: Reader stopping...")
+      @stopped = true
+      @selector.wakeup
+      @subscriber_mutex.synchronize do
+        @subscribers.each_value {|subscriber| subscriber.stop}
+      end
+    end
+
+    def add_timeout(interval, &block)
+      @timer.add(interval, &block)
+    end
+
+    def to_s
+      @name
+    end
+
+    def s_to_a(val)
+      val.kind_of?(String) ? [val] : val
+    end
+  end
+end

data/lib/nsq/subscriber.rb

@@ -0,0 +1,121 @@
+module NSQ
+  class Subscriber
+    attr_reader :selector, :name
+    attr_accessor :max_in_flight
+
+    def initialize(reader, topic, channel, options, &block)
+      options          = reader.options.merge(options)
+      @name            = "#{reader.name}:#{topic}:#{channel}"
+      @reader          = reader
+      @selector        = reader.selector
+      @topic           = topic
+      @channel         = channel
+      @block           = block
+      @max_tries       = options[:max_tries]
+      @max_in_flight   = (options[:max_in_flight] || 1).to_i
+      @requeue_delay   = (options[:requeue_delay] || 90).to_i * 1000
+      @connection_hash = {}
+
+      ready_options      = options[:ready_backoff_timer]           || {}
+      connection_options = options[:connection_backoff_timer]      || {}
+
+      @ready_min_interval      = ready_options[:min_interval]      || 0
+      @ready_max_interval      = ready_options[:max_interval]      || 120
+      @ready_ratio             = ready_options[:ratio]             || 0.25
+      @ready_short_length      = ready_options[:short_length]      || 10
+      @ready_long_length       = ready_options[:long_length]       || 250
+
+      @connection_min_interval = connection_options[:min_interval] || 0
+      @connection_max_interval = connection_options[:max_interval] || 30
+      @connection_ratio        = connection_options[:ratio]        || 0.25
+      @connection_short_length = connection_options[:short_length] || 10
+      @connection_long_length  = connection_options[:long_length]  || 250
+
+      raise "Invalid value for max_in_flight, must be between 0 and 2500: #{@max_in_flight}" unless @max_in_flight.between?(1,2499)
+    end
+
+    def create_ready_backoff_timer
+      BackoffTimer.new(@ready_min_interval, @ready_max_interval, @ready_ratio, @ready_short_length, @ready_long_length)
+    end
+
+    def create_connection_backoff_timer
+      BackoffTimer.new(@connection_min_interval, @connection_max_interval, @connection_ratio, @connection_short_length, @connection_long_length)
+    end
+
+    # Threshold for a connection where it's time to send a new READY message
+    def ready_threshold
+      @max_in_flight / @connection_hash.size / 4
+    end
+
+    # The actual value for the READY message
+    def ready_count
+      # TODO: Should we take into account the last_ready_count minus the number of messages sent since then?
+      # Rounding up!
+      (@max_in_flight + @connection_hash.size - 1) / @connection_hash.size
+    end
+
+    def connection_count
+      @connection_hash.size
+    end
+
+    def add_connection(host, port)
+      @connection_hash[[host, port]] = Connection.new(@reader, self, host, port)
+    end
+
+    def remove_connection(host, port)
+      connection = @connection_hash.delete([host, port])
+      return unless connection
+      connection.close
+    end
+
+    def stop
+      @stopped = true
+      @connection_hash.each_value do |connection|
+        connection.close
+      end
+      @connection_hash.clear
+    end
+
+    def stopped?
+      @stopped
+    end
+
+    def handle_connection(connection)
+      connection.send_init(@topic, @channel, @reader.short_id, @reader.long_id)
+    end
+
+    def handle_heartbeat(connection)
+    end
+
+    def handle_message(connection, message)
+      process_message(connection, message, &@block)
+    end
+
+    def process_message(connection, message, &block)
+      yield message
+      connection.send_finish(message.id, true)
+    rescue Exception => e
+      NSQ.logger.error("#{connection.name}: Exception during handle_message: #{e.message}\n\t#{e.backtrace.join("\n\t")}")
+      if @max_tries && attempts >= @max_tries
+        NSQ.logger.warning("#{connection.name}: Giving up on message after #{@max_tries} tries: #{body.inspect}")
+        connection.send_finish(message.id, false)
+      else
+        connection.send_requeue(message.id, attempts * @requeue_delay)
+      end
+    end
+
+    def handle_frame_error(connection, error_message)
+      NSQ.logger.error("Received error from nsqd: #{error_message.inspect}")
+      connection.reset
+    end
+
+    def handle_io_error(connection, exception)
+      NSQ.logger.error("Socket error: #{exception.message}\n\t#{exception.backtrace[0,2].join("\n\t")}")
+      connection.reset
+    end
+
+    def to_s
+      @name
+    end
+  end
+end

data/lib/nsq/timer.rb

@@ -0,0 +1,39 @@
+require 'thread'
+
+module NSQ
+  class Timer
+    def initialize(selector)
+      @selector   = selector
+      @proc_array = []
+      @mutex      = Mutex.new
+    end
+
+    def add(interval, &block)
+      new_run_at = Time.now + interval
+      @mutex.synchronize do
+        old_next_pair = @proc_array.first
+        @proc_array << [new_run_at, block]
+        # Sort the proc_array so the next one to run is at the front
+        @proc_array.sort_by { |pair| pair.first }
+        new_next_pair = @proc_array.first
+        # If the next proc has changed, then wakeup the selector so we can set the new next time
+        @selector.wakeup unless new_next_pair == old_next_pair
+      end
+    end
+
+    # Execute any necessary procs and return the next interval or nil if no procs
+    def next_interval
+      now = Time.now
+      @mutex.synchronize do
+        loop do
+          run_at, proc = @proc_array.first
+          return nil unless run_at
+          interval = run_at - now
+          return interval if interval > 0
+          proc.call
+          @proc_array.shift
+        end
+      end
+    end
+  end
+end

data/lib/nsq.rb

@@ -0,0 +1,35 @@
+require 'nsq/loggable'
+require 'nsq/message'
+require 'nsq/reader'
+require 'nsq/subscriber'
+require 'nsq/queue_subscriber'
+require 'nsq/connection'
+require 'nsq/backoff_timer'
+require 'nsq/timer'
+
+module NSQ
+  extend NSQ::Loggable
+
+  MAGIC_V2 = "  V2"
+
+  FRAME_TYPE_RESPONSE = 0
+  FRAME_TYPE_ERROR    = 1
+  FRAME_TYPE_MESSAGE  = 2
+
+  def self.create_reader(options, &block)
+    Reader.new(options, &block)
+  end
+
+  def self.assert_topic_and_channel_valid(topic, channel)
+    raise "Invalid topic #{topic}" unless valid_topic_name?(topic)
+    raise "Invalid channel #{channel}" unless valid_channel_name?(channel)
+  end
+
+  def self.valid_topic_name?(topic)
+    !!topic.match(/^[\.a-zA-Z0-9_-]+$/)
+  end
+
+  def self.valid_channel_name?(channel)
+    !!channel.match(/^[\.a-zA-Z0-9_-]+(#ephemeral)?$/)
+  end
+end

data/lib/ruby_nsq.rb

@@ -0,0 +1 @@
+require 'nsq'

data/test/backoff_timer_test.rb

@@ -0,0 +1,27 @@
+require 'minitest/autorun'
+require 'nsq/backoff_timer'
+
+describe ::NSQ::BackoffTimer do
+  before do
+    @timer = ::NSQ::BackoffTimer.new(0.1, 120, 0.25, 10, 1000)
+  end
+
+  describe "" do
+    it "should return the proper interval on successes and failures" do
+      assert_equal '%0.4f' % @timer.interval, '0.1000', @timer.inspect
+      @timer.success
+      assert_equal '%0.4f' % @timer.interval, '0.1000', @timer.inspect
+      @timer.failure
+      assert_equal '%0.2f' % @timer.interval, '3.19', @timer.inspect
+      assert_equal '%0.4f' % @timer.min_interval, '0.1000', @timer.inspect
+      assert_equal '%0.4f' % @timer.short_interval, '2.9975', @timer.inspect
+      assert_equal '%0.6f' % @timer.long_interval, '0.089925', @timer.inspect
+      @timer.failure
+      assert_equal '%0.2f' % @timer.interval, '6.27', @timer.inspect
+      @timer.success
+      assert_equal '%0.2f' % @timer.interval, '3.19', @timer.inspect
+      25.times { @timer.failure }
+      assert_equal '%0.2f' % @timer.interval, '32.41', @timer.inspect
+    end
+  end
+end

data/test/sync_connection_test.rb

@@ -0,0 +1,12 @@
+
+c = SyncConn()
+c.connect("127.0.0.1", 4150)
+c.send(nsq.subscribe('test', 'ch', 'a', 'b'))
+10.times do
+  c.send(nsq.ready(1))
+  resp = c.read_response()
+  unpacked = nsq.unpack_response(resp)
+  msg = nsq.decode_message(unpacked[1])
+  print msg.id, msg.body
+  c.send(nsq.finish(msg.id))
+end

metadata

@@ -0,0 +1,168 @@
+--- !ruby/object:Gem::Specification
+name: ruby_nsq
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+  prerelease: 
+platform: ruby
+authors:
+- Brad Pardee
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2012-10-26 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: nio4r
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rdoc
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: minitest
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: turn
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: wirble
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: hirb
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby client for NSQ modeled after pynsq
+email:
+- bradpardee@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- lib/nsq/backoff_timer.rb
+- lib/nsq/connection.rb
+- lib/nsq/loggable.rb
+- lib/nsq/message.rb
+- lib/nsq/queue_subscriber.rb
+- lib/nsq/reader.rb
+- lib/nsq/subscriber.rb
+- lib/nsq/timer.rb
+- lib/nsq.rb
+- lib/ruby_nsq.rb
+- examples/async/reader.rb
+- examples/async/writer.rb
+- examples/simple/reader.rb
+- LICENSE.txt
+- Rakefile
+- History.md
+- README.md
+- test/backoff_timer_test.rb
+- test/sync_connection_test.rb
+homepage: http://github.com/ClarityServices/ruby_nsq
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1380563601085235267
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+      segments:
+      - 0
+      hash: 1380563601085235267
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Ruby client for NSQ
+test_files:
+- test/backoff_timer_test.rb
+- test/sync_connection_test.rb