propono 0.6.3 → 0.7.0

checksums.yaml

@@ -1,7 +1,15 @@
 ---
-SHA1:
-  metadata.gz: 299526d5c3416f58858496b984ec04ff96a74a69
-  data.tar.gz: 8ab281343f74f9c5bfc233891311f1f21ebbf9e7
+!binary "U0hBMQ==":
+  metadata.gz: !binary |-
+    MTk2ZTQ1YmMxYTg4NzdiOWEyZDQ1ZWQyOTI4N2QyMGE4ZTA5OWRiOQ==
+  data.tar.gz: !binary |-
+    ZmUwY2I2M2E4ZjQ4ZTAxYzIwNzI4OTVjNWQ1OTkyYmFhYmM4Njk4OA==
 SHA512:
-  metadata.gz: 9b3f4bbc2284bf4fad16d8c57dff469d94522b46948a5d1c882d0994b03a4c1d699680615dd653a0325671f512d84f47c165314d0e4d7cc914a8eda7871b318a
-  data.tar.gz: 16609ea0f80a7cb985b84d4b2015bdbedd2aa83441f6dbcab2263db623bc17d412c2f0a191b0aab7d7b178ca781c98baa181a7aaba18291260a333725333c479
+  metadata.gz: !binary |-
+    ZTI1OWEzZGUwZTMzNzllNTYyNTE0NGZmOTZiOWUxZjJlYzUyNzY1NmQzMGZk
+    NTExYzYzNjE5NmI4NGE1Y2ZkODEzMzNkNzM2NzI1ZDAyMDFjYjQ0ZWIxYzhl
+    OGQ2NzBhZDlmNWQxZGYxMzUyMDJkN2I5YTAwMGM1YjIwMzYwMzU=
+  data.tar.gz: !binary |-
+    ZjFhYzMxYzgxNmZmNWM5MmQ1ODkwMTJlYzY4N2ZmNTE2NTMwYTY3MTA3YWVk
+    YjAxYjEyZjgwMjA4MmMxNzZjMDk5ZDA2M2RlZDc5ZmExNzRlNzQ2MTk2Mjc0
+    ZDY2OWU0ZjI2MmE2NWFjOWEyZDY5MGQyMTIwNTdhNWViOTI1Zjc=

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+# 0.7.0 / Unreleased
+
+* [FEATURE] Add TCP publish and listen methods.
+
 # 0.6.3 / 2013-10-20
 
 * [FEATURE] Catch all StandardError exceptions for UDP publishes.

data/README.md

@@ -58,34 +58,60 @@ end
 ```
 In the background, Propono is automatically setting up a queue using SQS, a notification system using SNS, and glueing them all together for you. But you don't have to worry about any of that.
 
+### Using TCP for messages
+
+Publishing directly to SNS takes about 15x longer than publishing over a simple TCP connection. It is therefore some times favourable to publish to a seperate machine listening for TCP messages, which will then proxy them on.
+
+To send messages this way, you need to set up a little extra config:
+
+```ruby
+Propono.config.tcp_host = "some.host.running.a.propono.listener"
+Propono.config.tcp_port = 12543
+```
+
+You then simply pass the `:tcp` protocol into `publish`
+
+```ruby
+Propono.publish('some-topic', message, protocol: :tcp)
+```
+
+You'll now need another application running Propono to listen to the TCP feed. You can use the same machine or a different one, just make sure the port config is the same in both applications, and you're good to go.
+
+```ruby
+Propono.listen_to_tcp do |topic, message|
+  Propono.publish(topic, message) # Proxy the message to SNS
+end
+```
+
+This proxying of TCP to SQS is used so often that there's a simple shortcut. Just run this on the machine receiving the TCP packets.
+
+```ruby
+Propono.proxy_tcp()
+```
+
 ### Using UDP for messages
 
 If you want almost-zero performance impact, and don't mind the occasional message getting lost, you can use UDP. We use this for things like our live dashboard where we don't mind losing a piece of activity here and there, but any perforamnce impact on our Meducation itself is bad news.
 
-To send messages this way, you need to set up a little extra config:
+Sending messages in this way is very similar to using TCP. First add some config:
 
 ```ruby
 Propono.config.udp_host = "some.host.running.a.propono.listener"
 Propono.config.udp_port = 12543
 ```
 
-You then simply pass the `:udp` protocol into `publish`
+You then simply pass the `:udp` protocol into `publish`:
 
 ```ruby
 Propono.publish('some-topic', message, protocol: :udp)
-```
 
-You'll now need another application running Propono to listen to the UDP feed. You can use the same machine or a different one, just make sure the port config is the same in both applications, and you're good to go.
+As per the `listen_to_tcp` method explained above, you now listen to udp or use the proxy method:
 
 ```ruby
 Propono.listen_to_udp do |topic, message|
   Propono.publish(topic, message) # Proxy the message to SNS
 end
-```
-
-This proxying of UDP to SQS is used so often that there's a simple shortcut. Just run this on the machine receiving the UDP packets.
 
-```ruby
 Propono.proxy_udp()
 ```
 
@@ -100,7 +126,9 @@ Propono.config do |config|
   config.queue_region = "An AWS queue region"
   config.application_name = "A name unique in your network"
   config.udp_host = "The host of a machine used for UDP proxying"
-  config.udp_port = "The host of a machine used for UDP proxying"
+  config.udp_port = "The port of a machine used for UDP proxying"
+  config.tcp_host = "The host of a machine used for TCP proxying"
+  config.tcp_port = "The port of a machine used for TCP proxying"
   config.logger = "A logger such as Log4r or Rails.logger"
 end
 ```

data/lib/propono.rb

@@ -20,6 +20,7 @@ require "propono/services/queue_listener"
 require "propono/services/subscriber"
 require "propono/services/topic_creator"
 require "propono/services/udp_listener"
+require "propono/services/tcp_listener"
 
 # Propono is a pub/sub gem built on top of Amazon Web Services (AWS).
 # It uses Simple Notification Service (SNS) and Simple Queue Service (SQS)
@@ -113,6 +114,17 @@ module Propono
     UdpListener.listen(&message_processor)
   end
 
+  # Listens for TCP messages and yields for each.
+  #
+  # Calling this will enter a queue-listening loop that
+  # yields the message_processor for each UDP message received.
+  #
+  # @param &message_processor The block to yield for each message.
+  #   Is called with <tt>|topic, message|</tt>.
+  def self.listen_to_tcp(&message_processor)
+    TcpListener.listen(&message_processor)
+  end
+
   # Listens for UDP messages and passes them onto the queue.
   #
   # This method uses #listen_to_udp and #publish to proxy
@@ -122,4 +134,14 @@ module Propono
       Propono.publish(topic, message)
     end
   end
+
+  # Listens for TCP messages and passes them onto the queue.
+  #
+  # This method uses #listen_to_tcp and #publish to proxy
+  # messages from TCP onto the queue.
+  def self.proxy_tcp
+    Propono.listen_to_tcp do |topic, message|
+      Propono.publish(topic, message)
+    end
+  end
 end

data/lib/propono/configuration.rb

@@ -9,6 +9,7 @@ module Propono
       :access_key, :secret_key, :queue_region,
       :application_name,
       :udp_host, :udp_port,
+      :tcp_host, :tcp_port,
       :logger
     ]
     attr_writer *SETTINGS

data/lib/propono/services/publisher.rb

@@ -1,3 +1,5 @@
+require 'socket'
+
 module Propono
   class PublisherError < ProponoError
   end
@@ -38,5 +40,15 @@ module Propono
     rescue => e
       Propono.config.logger.error "Propono failed to send : #{e}"
     end
+
+    def publish_via_tcp
+      payload = {topic: topic_id, message: message}.to_json
+
+      socket = TCPSocket.new(Propono.config.tcp_host, Propono.config.tcp_port)
+      socket.write payload
+      socket.close
+    rescue => e
+      Propono.config.logger.error "Propono failed to send : #{e}"
+    end
   end
 end

data/lib/propono/services/tcp_listener.rb

@@ -0,0 +1,40 @@
+require 'json'
+
+module Propono
+  class TcpListenerError < ProponoError
+  end
+
+  class TcpListener
+
+    def self.listen(&processor)
+      new(&processor).listen
+    end
+
+    def initialize(&processor)
+      raise TcpListenerError.new("Please provide a block to call for each message") unless block_given?
+      @processor = processor
+    end
+
+    def listen
+      loop { receive_and_process }
+    end
+
+    private
+
+    def receive_and_process
+      client = server.accept
+      tcp_data = client.recvfrom(1024)[0]
+      client.close
+      Thread.new { process_tcp_data(tcp_data) }
+    end
+
+    def process_tcp_data(tcp_data)
+      json = JSON.parse(tcp_data)
+      @processor.call(json['topic'], json['message'])
+    end
+
+    def server
+      @server ||= TCPServer.open(Propono.config.tcp_port)
+    end
+  end
+end

data/lib/propono/version.rb

@@ -1,3 +1,3 @@
 module Propono
-  VERSION = "0.6.3"
+  VERSION = "0.7.0"
 end

data/test/configuration_test.rb

@@ -43,6 +43,30 @@ module Propono
       assert_equal application_name, Propono.config.application_name
     end
 
+    def test_udp_host
+      val = "test-application-name"
+      Propono.config.udp_host = val
+      assert_equal val, Propono.config.udp_host
+    end
+
+    def test_udp_port
+      val = 10000
+      Propono.config.udp_port = val
+      assert_equal val, Propono.config.udp_port
+    end
+
+    def test_tcp_host
+      val = "test-application-name"
+      Propono.config.tcp_host = val
+      assert_equal val, Propono.config.tcp_host
+    end
+
+    def test_tcp_port
+      val = 9382
+      Propono.config.tcp_port = val
+      assert_equal val, Propono.config.tcp_port
+    end
+
     def test_missing_access_key_throws_exception
       assert_raises(ProponoConfigurationError) do
         Propono.config.access_key

data/test/integration/sns_to_sqs_test.rb

@@ -14,6 +14,9 @@ module Propono
           break
         end
       end
+
+      sleep(2) # Make sure the listener has started
+
       Propono.publish(topic, text)
       flunk("Test Timeout") unless wait_for_thread(thread)
     ensure

data/test/integration/tcp_to_sqs_test.rb

@@ -0,0 +1,35 @@
+require File.expand_path('../integration_test', __FILE__)
+
+module Propono
+  class UdpToSqsTest < IntegrationTest
+    def test_the_message_gets_there
+      topic = "test-topic"
+      message = "This is my message"
+      Propono.config.tcp_host = "localhost"
+      Propono.config.tcp_port = 20009
+
+      Propono.subscribe_by_queue(topic)
+
+      sqs_thread = Thread.new do
+        Propono.listen_to_queue(topic) do |sqs_message|
+          assert_equal message, sqs_message
+          sqs_thread.terminate
+        end
+      end
+
+      tcp_thread = Thread.new do
+        Propono.listen_to_tcp do |tcp_topic, tcp_message|
+          Propono.publish(tcp_topic, tcp_message)
+          tcp_thread.terminate
+        end
+      end
+      sleep(2) # Make sure the listener has started
+
+      Propono.publish(topic, message, protocol: :tcp)
+      flunk("Test Timeout") unless wait_for_thread(tcp_thread) && wait_for_thread(sqs_thread)
+    ensure
+      tcp_thread.terminate
+      sqs_thread.terminate
+    end
+  end
+end

data/test/integration/udp_proxy_test.rb

@@ -9,10 +9,6 @@ module Propono
 
       Propono.subscribe_by_queue(topic)
 
-      udp_thread = Thread.new do
-        Propono.proxy_udp
-      end
-
       sqs_thread = Thread.new do
         Propono.listen_to_queue(topic) do |sqs_message|
           assert_equal message, sqs_message
@@ -20,6 +16,12 @@ module Propono
         end
       end
 
+      udp_thread = Thread.new do
+        Propono.proxy_udp
+      end
+
+      sleep(2) # Make sure the proxy has started
+
       Propono.publish(topic, message, protocol: :udp)
       flunk("Test timeout") unless wait_for_thread(sqs_thread)
     ensure

data/test/integration/udp_to_sqs_test.rb

@@ -9,6 +9,13 @@ module Propono
 
       Propono.subscribe_by_queue(topic)
 
+      sqs_thread = Thread.new do
+        Propono.listen_to_queue(topic) do |sqs_message|
+          assert_equal message, sqs_message
+          sqs_thread.terminate
+        end
+      end
+
       udp_thread = Thread.new do
         Propono.listen_to_udp do |udp_topic, udp_message|
           Propono.publish(udp_topic, udp_message)
@@ -16,12 +23,7 @@ module Propono
         end
       end
 
-      sqs_thread = Thread.new do
-        Propono.listen_to_queue(topic) do |sqs_message|
-          assert_equal message, sqs_message
-          sqs_thread.terminate
-        end
-      end
+      sleep(2) # Make sure the listener has started
 
       Propono.publish(topic, message, protocol: :udp)
       flunk("Test Timeout") unless wait_for_thread(udp_thread) && wait_for_thread(sqs_thread)

data/test/propono_test.rb

@@ -32,6 +32,11 @@ module Propono
       Propono.listen_to_udp()
     end
 
+    def test_listen_to_tcp_calls_tcp_listener
+      TcpListener.expects(:listen).with()
+      Propono.listen_to_tcp()
+    end
+
     def test_proxy_udp_calls_listen
       UdpListener.expects(:listen).with()
       Propono.proxy_udp()
@@ -44,5 +49,18 @@ module Propono
       Publisher.expects(:publish).with(topic, message, {})
       Propono.proxy_udp
     end
+
+    def test_proxy_tcp_calls_listen
+      TcpListener.expects(:listen).with()
+      Propono.proxy_tcp()
+    end
+
+    def test_proxy_tcp_calls_publish_in_the_block
+      topic = "foobar"
+      message = "message"
+      Propono.stubs(:listen_to_tcp).yields(topic, message)
+      Publisher.expects(:publish).with(topic, message, {})
+      Propono.proxy_tcp
+    end
   end
 end

data/test/services/publisher_test.rb

@@ -122,6 +122,52 @@ module Propono
       end
     end
 
+    def test_tcp_uses_correct_message
+      Propono.config.tcp_host = "http://meducation.net"
+      Propono.config.tcp_port = 1234
+      topic_id = "my-fav-topic"
+      message = "foobar"
+      payload = {topic: topic_id, message: message}.to_json
+
+      socket = mock()
+      socket.expects(:write).with(payload)
+      socket.expects(:close)
+      TCPSocket.stubs(new: socket)
+
+      publisher = Publisher.new(topic_id, message)
+      publisher.send(:publish_via_tcp)
+    end
+
+    def test_tcp_uses_correct_message_host_and_port
+      host = "http://meducation.net"
+      port = 1234
+      Propono.config.tcp_host = host
+      Propono.config.tcp_port = port
+      topic_id = "my-fav-topic"
+      message = "foobar"
+      TCPSocket.expects(:new).with(host, port)
+
+      publisher = Publisher.new(topic_id, message)
+      publisher.send(:publish_via_tcp)
+    end
+
+    def test_exception_from_tcpsocket_caught_and_logged
+      host = "http://meducation.net"
+      port = 1234
+      Propono.config.tcp_host = host
+      Propono.config.tcp_port = port
+
+      client = Publisher.new("topic_id", "message")
+      Propono.config.logger.expects(:error).with() {|x| x =~ /^Propono failed to send : getaddrinfo:.*/}
+      client.send(:publish_via_tcp)
+    end
+
+    def test_publish_should_raise_exception_if_topic_is_nil
+      assert_raises(PublisherError, "Topic is nil") do
+        Publisher.publish(nil, "foobar")
+      end
+    end
+
     def test_publish_should_raise_exception_if_message_is_nil
       assert_raises(PublisherError, "Message is nil") do
         Publisher.publish("foobar", nil)

data/test/services/tcp_listener_test.rb

@@ -0,0 +1,63 @@
+require File.expand_path('../../test_helper', __FILE__)
+
+module Propono
+  class TcpListenerTest < Minitest::Test
+
+    def test_intialize_sets_locals
+      block = Proc.new {}
+      listener = TcpListener.new(&block)
+      assert_equal block, listener.instance_variable_get("@processor")
+    end
+
+    def test_server_is_setup_correctly
+      port = 1234
+      Propono.config.tcp_port = port
+
+      TCPServer.expects(:open).with(port)
+
+      listener = TcpListener.new() {}
+      server = listener.send(:server)
+    end
+
+    def test_initialize_should_fail_without_a_block
+      assert_raises(TcpListenerError) do
+        TcpListener.new
+      end
+    end
+
+    def test_message_is_processed
+      tcp_msg = "Foobar"
+      processor = Proc.new {}
+      listener = TcpListener.new(&processor)
+      client = mock()
+      client.expects(:recvfrom => [tcp_msg])
+      client.expects(:close)
+
+      server = mock()
+      server.expects(accept: client)
+
+      listener.stubs(server: server)
+      listener.expects(:process_tcp_data).with(tcp_msg)
+      thread = listener.send(:receive_and_process)
+      thread.join
+    end
+
+    def test_processor_is_called_correctly
+      topic = "my-topic"
+      message = "my-message"
+      processor = Proc.new {}
+      tcp_data = {topic: topic, message: message}.to_json
+      processor.expects(:call).with(topic, message)
+
+      listener = TcpListener.new(&processor)
+      listener.send(:process_tcp_data, tcp_data)
+    end
+
+    def test_listen_should_loop
+      listener = TcpListener.new {}
+      listener.expects(:loop)
+      listener.listen
+    end
+  end
+end
+

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: propono
 version: !ruby/object:Gem::Version
-  version: 0.6.3
+  version: 0.7.0
 platform: ruby
 authors:
 - MalcyL
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-10-20 00:00:00.000000000 Z
+date: 2013-10-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: fog
@@ -43,42 +43,42 @@ dependencies:
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: mocha
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: yard
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -125,6 +125,7 @@ files:
 - lib/propono/services/queue_creator.rb
 - lib/propono/services/queue_listener.rb
 - lib/propono/services/subscriber.rb
+- lib/propono/services/tcp_listener.rb
 - lib/propono/services/topic_creator.rb
 - lib/propono/services/udp_listener.rb
 - lib/propono/version.rb
@@ -139,6 +140,7 @@ files:
 - test/configuration_test.rb
 - test/integration/integration_test.rb
 - test/integration/sns_to_sqs_test.rb
+- test/integration/tcp_to_sqs_test.rb
 - test/integration/udp_proxy_test.rb
 - test/integration/udp_to_sqs_test.rb
 - test/logger_test.rb
@@ -147,6 +149,7 @@ files:
 - test/services/queue_creator_test.rb
 - test/services/queue_listener_test.rb
 - test/services/subscriber_test.rb
+- test/services/tcp_listener_test.rb
 - test/services/topic_creator_test.rb
 - test/services/udp_listener_test.rb
 - test/test_helper.rb
@@ -160,12 +163,12 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
@@ -185,6 +188,7 @@ test_files:
 - test/configuration_test.rb
 - test/integration/integration_test.rb
 - test/integration/sns_to_sqs_test.rb
+- test/integration/tcp_to_sqs_test.rb
 - test/integration/udp_proxy_test.rb
 - test/integration/udp_to_sqs_test.rb
 - test/logger_test.rb
@@ -193,6 +197,7 @@ test_files:
 - test/services/queue_creator_test.rb
 - test/services/queue_listener_test.rb
 - test/services/subscriber_test.rb
+- test/services/tcp_listener_test.rb
 - test/services/topic_creator_test.rb
 - test/services/udp_listener_test.rb
 - test/test_helper.rb