htttee 0.5.0 → 0.6.0

data/.gitignore

@@ -2,5 +2,4 @@
 .DS_Store
 
 Gemfile.lock
-pkg
-vendor
+pkg

data/.travis.yml

@@ -0,0 +1,6 @@
+language: ruby
+rvm:
+  - "1.9.2"
+  - "1.9.3"
+  - "2.0.0"
+  - rbx-19mode

data/README.md

@@ -1,5 +1,7 @@
 # htttee - unix's tee-as-a-service
 
+[![Build Status](https://travis-ci.org/benburkert/htttee.png)](https://travis-ci.org/benburkert/htttee)
+
 
 ## What is 'tee'?
 
@@ -29,7 +31,7 @@ In one terminal:
 
 In another terminal:
 
-    curl http://htttee.engineyard.com/SOMEUNIQUESTRING
+    curl http://localhost:3000/SOMEUNIQUESTRING
 
 Or to see the chunked information:
 
@@ -56,6 +58,16 @@ final few numbers chunked through:
     
     Connection closed by foreign host.
 
+## Browser Support
+
+Some browsers don't behave well when recieving a chunked response of plain/text data. For example,
+lots of them buffer the first 256 bytes or so before doing anything. [SSE](http://www.html5rocks.com/en/tutorials/eventsource/basics/)
+is used to get around this browser limitation. If the request for a stream originates from a browser
+that supports SSE and the request is for 'text/html' then SSE setup page is returned. That page
+has a little javascript for setting up the stream and parsing the events. The simplicity of the SSE
+protocol makes it difficult to sanely send newline characters. So there is a `ctrl` event has been
+added that makes it easy to send newline characters and other 'special' characters. This could even
+allow for terminal emulation in the future.
 
 ## Running the server
 

data/bin/htttee

@@ -24,7 +24,7 @@ end
 opts = Trollop::options do
   opt :uuid, "The UUID of the stream.", :short => '-u', :type => String
   opt :'content-type', "The content-type of the stream.", :short => '-c', :type => String, :default => 'text/plain'
-  opt :endpoint, "The endpoint of the htttee service.", :short => '-e', :type => String, :default => 'http://htttee.engineyard.com/'
+  opt :endpoint, "The endpoint of the htttee service.", :short => '-e', :type => String, :default => 'http://localhost:3000/'
   opt :quiet, "Don't echo to stdout.", :short => '-q', :default => false
 end
 
@@ -34,6 +34,6 @@ $stdin.sync  = true
 $stdout.sync = true
 
 input  = opts[:quiet] ? $stdin : MultiplexedIO.new($stdin, $stdout)
-client = EY::Tea::Client.new(:endpoint => opts[:endpoint])
+client = HTTTee::Client.new(:endpoint => opts[:endpoint])
 
 client.up(input, opts[:uuid], opts[:'content-type'])

data/bin/htttee-exec

@@ -1,16 +1,17 @@
 #!/bin/sh
 
-stdout_uuid="$(head -c4096 /dev/urandom | sha256sum | awk '{ print $1 }')"
-stderr_uuid="$(head -c4096 /dev/urandom | sha256sum | awk '{ print $1 }')"
+set -e
+
+stdout_uuid="$(openssl rand -hex 16)"
+stderr_uuid="$(openssl rand -hex 16)"
 
-mkfifo $stdout_uuid
 mkfifo $stderr_uuid
 
-bin/htttee --uuid $stdout_uuid --endpoint http://127.0.0.1:9292/ < $stdout_uuid &
-bin/htttee --uuid $stderr_uuid --endpoint http://127.0.0.1:9292/ < $stderr_uuid &
+~/code/htttee/stubs/htttee --uuid $stdout_uuid --endpoint http://127.0.0.1:3000/ < $stdout_uuid &
+~/code/htttee/stubs/htttee --uuid $stderr_uuid --endpoint http://127.0.0.1:3000/ < $stderr_uuid &
 
-echo "stdout: http://127.0.0.1:9292/$stdout_uuid"
-echo "stderr: http://127.0.0.1:9292/$stderr_uuid"
+echo "stdout: http://127.0.0.1:3000/$stdout_uuid" >&2
+echo "stderr: http://127.0.0.1:3000/$stderr_uuid" >&2
 
 exec 1>$stdout_uuid
 exec 2>$stderr_uuid

data/config.ru

@@ -3,4 +3,4 @@ $:.unshift File.join(File.dirname(__FILE__), 'lib')
 require 'htttee/server'
 
 use Rack::CommonLogger
-run EY::Tea::Server.app
+run HTTTee::Server.app

data/htttee.gemspec

@@ -1,6 +1,8 @@
+require File.join(File.dirname(__FILE__), 'lib', 'htttee', 'version')
+
 Gem::Specification.new do |s|
   s.name        = "htttee"
-  s.version     = '0.5.0'
+  s.version     = HTTTee::VERSION
   s.platform    = Gem::Platform::RUBY
   s.authors     = ["Ben Burkert"]
   s.email       = ["ben@benburkert.com"]
@@ -15,7 +17,7 @@ Gem::Specification.new do |s|
   s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
   s.require_paths = ["lib"]
 
-  s.add_dependency 'rack-client', '~> 0.3.1.pre.i'
+  s.add_dependency 'rack-client', '>= 0.4.2'
   s.add_dependency 'trollop'
 
   s.add_development_dependency 'sinatra'
@@ -26,5 +28,4 @@ Gem::Specification.new do |s|
 
   s.add_development_dependency 'rspec'
   s.add_development_dependency 'rake'
-  # s.add_development_dependency 'ruby-debug19'
 end

data/lib/htttee/client.rb

@@ -2,12 +2,10 @@ require 'net/http'
 require 'rack/client'
 #require 'uuidtools'
 
-module EY
-  module Tea
-    module Client
-      def self.new(*a)
-        Consumer.new(*a)
-      end
+module HTTTee
+  module Client
+    def self.new(*a)
+      Consumer.new(*a)
     end
   end
 end

data/lib/htttee/client/consumer.rb

@@ -1,48 +1,52 @@
-module EY
-  module Tea
-    module Client
-      class Consumer < Rack::Client::Base
-        def initialize(options = {})
-          @base_uri = URI.parse(options.fetch(:endpoint, 'http://tea.engineyard.com/'))
-          inner_app = options.fetch(:app, Rack::Client::Handler::NetHTTP.new)
-
-          super(inner_app)
-        end
+module HTTTee
+  module Client
+    class Consumer < Rack::Client::Base
+      def initialize(options = {})
+        @base_uri = URI.parse(options.fetch(:endpoint, 'http://localhost:3000/'))
+        inner_app = options.fetch(:app, Rack::Client::Handler::NetHTTP.new)
+
+        super(inner_app)
+      end
 
-        def up(io, uuid, content_type = 'text/plain')
-          post("/#{uuid}", {'Content-Type' => content_type, 'Transfer-Encoding' => 'chunked'}, io)
-        end
+      def up(io, uuid, content_type = 'text/plain')
+        headers = {
+          'Content-Type'      => content_type,
+          'Transfer-Encoding' => 'chunked',
+          'Connection'        => 'Keep-Alive',
+        }
 
-        def down(uuid)
-          get("/#{uuid}") do |status, headers, response_body|
-            response_body.each do |chunk|
-              yield chunk
-            end
+        post("/#{uuid}", headers, io)
+      end
+
+      def down(uuid)
+        get("/#{uuid}") do |status, headers, response_body|
+          response_body.each do |chunk|
+            yield chunk
           end
         end
+      end
 
-        def build_env(request_method, url, headers = {}, body = nil)
-          uri = @base_uri.nil? ? URI.parse(url) : @base_uri + url
+      def build_env(request_method, url, headers = {}, body = nil)
+        uri = @base_uri.nil? ? URI.parse(url) : @base_uri + url
 
-          env = super(request_method, uri.to_s, headers, body)
+        env = super(request_method, uri.to_s, headers, body)
 
-          env['HTTP_HOST']       ||= http_host_for(uri)
-          env['HTTP_USER_AGENT'] ||= http_user_agent
+        env['HTTP_HOST']       ||= http_host_for(uri)
+        env['HTTP_USER_AGENT'] ||= http_user_agent
 
-          env
-        end
+        env
+      end
 
-        def http_host_for(uri)
-          if uri.to_s.include?(":#{uri.port}")
-            [uri.host, uri.port].join(':')
-          else
-            uri.host
-          end
+      def http_host_for(uri)
+        if uri.to_s.include?(":#{uri.port}")
+          [uri.host, uri.port].join(':')
+        else
+          uri.host
         end
+      end
 
-        def http_user_agent
-          "htttee (rack-client #{Rack::Client::VERSION} (app: #{@app.class}))"
-        end
+      def http_user_agent
+        "htttee (rack-client #{Rack::Client::VERSION} (app: #{@app.class}))"
       end
     end
   end

data/lib/htttee/server.rb

@@ -5,59 +5,61 @@ require 'em-redis'
 
 EM.epoll
 
-module EY
-  module Tea
-    module Server
+module HTTTee
+  module Server
 
-      def self.app
-        mocking? ? mock_app : rack_app
-      end
+    def self.app
+      mocking? ? mock_app : rack_app
+    end
 
-      def self.api(host = (ENV['REDIS_HOST'] || 'localhost' ), port = (ENV['REDIS_PORT'] || 6379).to_i)
-        Api.new(host, port)
-      end
+    def self.api(host = (ENV['REDIS_HOST'] || 'localhost' ), port = (ENV['REDIS_PORT'] || 6379).to_i)
+      Api.new(host, port)
+    end
 
-      def self.rack_app
-        Rack::Builder.app do |builder|
-          builder.use AsyncFixer
-          builder.use Dechunker
-          builder.run Server.api
-        end
+    def self.rack_app
+      Rack::Builder.app do |builder|
+        builder.use AsyncFixer
+        builder.use Dechunker
+        builder.use Rechunker
+        builder.use SSE::Middleware
+        builder.run Server.api
       end
+    end
 
-      def self.mock_app
-        Rack::Builder.app do |builder|
-          builder.use Mock::ThinMuxer
-          builder.use Mock::EchoUri
-          builder.run Server.rack_app
-        end
+    def self.mock_app
+      Rack::Builder.app do |builder|
+        builder.use Mock::ThinMuxer
+        builder.use Mock::EchoUri
+        builder.use Rechunker
+        builder.use SSE::Middleware
+        builder.run Server.rack_app
       end
+    end
 
-      def self.mock!
-        require 'htttee/server/mock'
-        @mocking = true
+    def self.mock!
+      require 'htttee/server/mock'
+      @mocking = true
 
-        @mock_uri = Mock.boot_forking_server
-      end
+      @mock_uri = Mock.boot_forking_server
+    end
 
-      def self.reset!
-        raise "Can't reset in non-mocked mode." unless mocking?
+    def self.reset!
+      raise "Can't reset in non-mocked mode." unless mocking?
 
-        EM.run do
-          EM::Protocols::Redis.connect.flush_all do
-            EM.stop
-          end
+      EM.run do
+        EM::Protocols::Redis.connect.flush_all do
+          EM.stop
         end
       end
+    end
 
-      def self.mocking?
-        @mocking
-      end
+    def self.mocking?
+      @mocking
+    end
 
-      def self.mock_uri
-        raise "Not in mock mode!" unless mocking?
-        @mock_uri
-      end
+    def self.mock_uri
+      raise "Not in mock mode!" unless mocking?
+      @mock_uri
     end
   end
 end
@@ -72,3 +74,7 @@ require 'htttee/server/chunked_body'
 
 require 'htttee/server/middleware/async_fixer'
 require 'htttee/server/middleware/dechunker'
+require 'htttee/server/middleware/rechunker'
+
+require 'htttee/server/sse/body'
+require 'htttee/server/sse/middleware'

data/lib/htttee/server/api.rb

@@ -1,200 +1,216 @@
-module EY
-  module Tea
-    module Server
-      class Api
-        STREAMING, FIN = ?0, ?1
-        SUBSCRIBE, UNSUBSCRIBE, MESSAGE = 'SUBSCRIBE', 'UNSUBSCRIBE', 'MESSAGE'
+module HTTTee
+  module Server
+    class Api
+      STREAMING, FIN = ?0, ?1
+      SUBSCRIBE, UNSUBSCRIBE, MESSAGE = 'SUBSCRIBE', 'UNSUBSCRIBE', 'MESSAGE'
 
-        AsyncResponse = [-1, {}, []].freeze
+      AsyncResponse = [-1, {}, []].freeze
 
-        attr_accessor :redis
+      attr_accessor :redis
 
-        def initialize(host, port)
-          @host, @port = host, port
-        end
-
-        def call(env)
-          uuid = env['PATH_INFO'].sub(/^\//, '')
-
-          case env['REQUEST_METHOD']
-          when 'POST' then post(env, uuid)
-          when 'GET'  then get(env, uuid)
-          end
-
-          AsyncResponse
-        end
-
-        def post(env, uuid)
-          body = Thin::DeferrableBody.new
+      def initialize(host, port)
+        @host, @port = host, port
+      end
 
-          redis.set(state_key(uuid), STREAMING)
+      def call(env)
+        uuid = env['PATH_INFO'].sub(/^\//, '')
+        body = env['rack.response_body']
 
-          set_input_callback(env, uuid, body)
-          set_input_errback(env, uuid, body)
-          set_input_each(env, uuid, body)
+        case env['REQUEST_METHOD']
+        when 'POST' then post(env, uuid, body)
+        when 'GET'  then get(env, uuid, body)
         end
 
-        def get(env, uuid)
-          body = ChunkedBody.new
+        AsyncResponse
+      end
 
-          with_state_for(uuid) do |state|
-            case state
-            when NilClass   then four_oh_four_response(env, body)
-            when STREAMING  then open_stream_response(env, uuid, body)
-            when FIN        then closed_stream_response(env, uuid, body)
-            end
-          end
-        end
+      def post(env, uuid, body)
+        redis.set(state_key(uuid), STREAMING)
 
-        def set_input_callback(env, uuid, body)
-          rack_input(env).callback do
-            async_callback(env).call [204, {}, body]
+        set_input_callback(env, uuid, body)
+        set_input_errback(env, uuid, body)
+        set_input_each(env, uuid, body)
+      end
 
-            redis.set(state_key(uuid), FIN) do
-              finish(channel(uuid))
-              body.succeed
-            end
+      def get(env, uuid, body)
+        with_state_for(uuid) do |state|
+          case state
+          when NilClass   then four_oh_four_response(env, body)
+          when STREAMING  then open_stream_response(env, uuid, body)
+          when FIN        then closed_stream_response(env, uuid, body)
           end
         end
+      end
 
-        def set_input_errback(env, uuid, body)
-          rack_input(env).errback do |error|
-            async_callback(env).call [500, {}, body]
+      def set_input_callback(env, uuid, body)
+        rack_input(env).callback do
+          async_callback(env).call [204, {}, body]
 
-            body.call [error.inspect]
+          redis.set(state_key(uuid), FIN) do
+            finish(channel(uuid))
             body.succeed
           end
         end
 
-        def set_input_each(env, uuid, body)
-          rack_input(env).each do |chunk|
-            unless chunk.empty?
-              redis.pipeline(['append', data_key(uuid), chunk], ['publish', channel(uuid), Yajl::Encoder.encode([STREAMING, chunk])])
-            end
+        async_close(env).callback do
+          redis.set(state_key(uuid), FIN) do
+            finish(channel(uuid))
           end
         end
+      end
 
-        def four_oh_four_response(env, body)
-          env['async.callback'].call [404, {'Transfer-Encoding' => 'chunked'}, body]
+      def set_input_errback(env, uuid, body)
+        rack_input(env).errback do |error|
+          async_callback(env).call [500, {}, body]
 
+          body.call [error.inspect]
           body.succeed
         end
+      end
 
-        def open_stream_response(env, uuid, body)
-          start_response(env, body)
-          stream_data_to(body, data_key(uuid)) do
-            with_state_for(uuid) do |state|
-              if state == FIN
-                body.succeed
-              else
-                subscribe_and_stream(env, uuid, body)
-              end
-            end
+      def set_input_each(env, uuid, body)
+        rack_input(env).each do |chunk|
+          unless chunk.empty?
+            redis.pipeline ['append', data_key(uuid), chunk],
+              ['publish', channel(uuid), encode(STREAMING, chunk)]
           end
         end
+      end
 
-        def closed_stream_response(env, uuid, body)
-          start_response(env, body)
-          stream_data_to(body, data_key(uuid)) do
-            body.succeed
-          end
-        end
+      def four_oh_four_response(env, body)
+        env['async.callback'].call [404, {'Transfer-Encoding' => 'chunked'}, body]
+
+        body.succeed
+      end
 
-        def stream_data_to(body, key, offset = 0, chunk_size = 1024, &block)
-          redis.substr(key, offset, offset + chunk_size) do |chunk|
-            if chunk.nil? || chunk.empty?
-              yield
+      def open_stream_response(env, uuid, body)
+        start_response(env, body)
+        stream_data_to(body, data_key(uuid)) do
+          with_state_for(uuid) do |state|
+            if state == FIN
+              body.succeed
             else
-              body.call [chunk]
-              stream_data_to(body, key, offset + chunk.size, chunk_size, &block)
+              subscribe_and_stream(env, uuid, body)
             end
           end
         end
+      end
 
-        def respond_with(env, body, data)
-          start_response(env, body, data)
+      def closed_stream_response(env, uuid, body)
+        start_response(env, body)
+        stream_data_to(body, data_key(uuid)) do
           body.succeed
         end
+      end
 
-        def start_response(env, body, data = nil)
-          env['async.callback'].call [200, {'Transfer-Encoding' => 'chunked', 'Content-Type' => 'text/plain'}, body]
-
-          body.call [data] unless data.nil? || data.empty?
+      def stream_data_to(body, key, offset = 0, chunk_size = 1024, &block)
+        redis.substr(key, offset, offset + chunk_size) do |chunk|
+          if chunk.nil? || chunk.empty?
+            yield
+          else
+            body.call [chunk]
+            stream_data_to(body, key, offset + chunk.size, chunk_size, &block)
+          end
         end
+      end
+
+      def respond_with(env, body, data)
+        start_response(env, body, data)
+        body.succeed
+      end
+
+      def start_response(env, body, data = nil)
+        env['async.callback'].call [200, {'Transfer-Encoding' => 'chunked', 'Content-Type' => 'text/plain'}, body]
 
-        def subscribe_and_stream(env, uuid, body)
-          subscribe channel(uuid) do |type, message, *extra|
-            case type
+        body.call [data] unless data.nil? || data.empty?
+      end
+
+      def subscribe_and_stream(env, uuid, body)
+        conn = subscribe channel(uuid) do |type, message, *extra|
+          case type
             #when SUBSCRIBE then start_response(env, body, data)
-            when FIN       then body.succeed
-            when STREAMING then body.call [message]
-            end
+          when FIN       then body.succeed
+          when STREAMING then body.call [message]
           end
         end
 
-        def with_state_for(uuid, &block)
-          redis.get(state_key(uuid), &block)
+        async_close(env).callback do
+          conn.close_connection
         end
+      end
 
-        def with_state_and_data_for(uuid, &block)
-          redis.multi_get(state_key(uuid), data_key(uuid), &block)
-        end
+      def with_state_for(uuid, &block)
+        redis.get(state_key(uuid), &block)
+      end
 
-        def data_key(uuid)
-          "#{uuid}:data"
-        end
+      def with_state_and_data_for(uuid, &block)
+        redis.multi_get(state_key(uuid), data_key(uuid), &block)
+      end
 
-        def state_key(uuid)
-          "#{uuid}:state"
-        end
+      def data_key(uuid)
+        "#{uuid}:data"
+      end
 
-        def channel(uuid)
-          uuid
-        end
+      def state_key(uuid)
+        "#{uuid}:state"
+      end
 
-        def rack_input(rack_env)
-          rack_env['rack.input']
-        end
+      def channel(uuid)
+        uuid
+      end
 
-        def async_callback(rack_env)
-          rack_env['async.callback']
-        end
+      def rack_input(rack_env)
+        rack_env['rack.input']
+      end
 
-        def publish(channel, data)
-          redis.publish channel, Yajl::Encoder.encode([STREAMING, data])
-        end
+      def async_callback(rack_env)
+        rack_env['async.callback']
+      end
 
-        def finish(channel)
-          redis.publish channel, Yajl::Encoder.encode([FIN])
-        end
+      def async_close(rack_env)
+        rack_env['async.close']
+      end
 
-        def subscribe(channel, &block)
-          conn = pubsub
-          conn.subscribe channel do |type, chan, data|
-            case type.upcase
-            when SUBSCRIBE then block.call(SUBSCRIBE, chan)
-            when MESSAGE
-              state, data = Yajl::Parser.parse(data)
-              case state
-              when STREAMING  then block.call(STREAMING, data)
-              when FIN
-                conn.unsubscribe channel
-                block.call(FIN, data)
-              end
-            else
-              debugger
-              ''
+      def publish(channel, data)
+        redis.publish channel, encode(STREAMING, data)
+      end
+
+      def finish(channel)
+        redis.publish channel, encode(FIN)
+      end
+
+      def subscribe(channel, &block)
+        conn = pubsub
+
+        conn.subscribe channel do |type, chan, data|
+          case type.upcase
+          when SUBSCRIBE then block.call(SUBSCRIBE, chan)
+          when MESSAGE
+            state, data = Yajl::Parser.parse(data)
+            case state
+            when STREAMING  then block.call(STREAMING, data)
+            when FIN
+              conn.unsubscribe channel
+              block.call(FIN, data)
             end
+          else
+            ''
           end
         end
 
-        def pubsub
-          EM::Protocols::PubSubRedis.connect(@host, @port)
-        end
+        conn
+      end
 
-        def redis
-          @redis ||= EM::Protocols::Redis.connect(@host, @port)
-        end
+      def pubsub
+        EM::Protocols::PubSubRedis.connect(@host, @port)
+      end
+
+      def redis
+        @@redis ||= EM::Protocols::Redis.connect(@host, @port)
+      end
+
+      def encode(*parts)
+        Yajl::Encoder.encode(parts)
       end
     end
   end