htttee 0.5.0

data/deploy/cookbooks/resque/templates/default/resque.yml.erb

@@ -0,0 +1,3 @@
+<%= @framework_env %>:
+  redis_host: <%= @redis_host %>
+  redis_port: <%= @redis_port %>

data/deploy/cookbooks/resque/templates/default/resque_scheduler.rb.erb

@@ -0,0 +1,73 @@
+rack_env   = "<%= @node[:environment][:framework_env] %>"
+app_name   = "<%= @app %>"
+app_root   = "<%= @app_root %>"
+owner      = "<%= @node[:owner_name] %>"
+home      = "/home/#{owner}"
+instance_id = "<%= @node.engineyard.id %>"
+
+<%= @resque_workers_count %>.times do |num|
+  inline = "#{home}/.ruby_inline/resque-#{app_name}-#{num}"
+
+  God.watch do |w|
+    w.name     = "resque-scheduler-#{app_name}-#{num}"
+    w.group    = "resque-#{app_name}"
+    w.uid      = owner
+    w.gid      = owner
+    w.interval = 30.seconds
+    w.log      = "#{app_root}/log/worker-scheduler.#{num}.log"
+    w.dir      = app_root
+    w.env      = {
+      "USER"        => owner,
+      "VERBOSE"     => "true",
+      "INSTANCE_ID" => instance_id,
+      "GOD_WATCH"   => w.name,
+      "RACK_ENV"    => rack_env,
+      "HOME"        => home,
+      "QUEUE"       => "*",
+    }
+
+    w.start    = "bundle exec rake --trace init resque:scheduler"
+
+    w.behavior(:clean_pid_file)
+
+    w.start_grace   = 2.minutes
+    w.restart_grace = 2.minutes
+
+    # retart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end

data/deploy/solo.rb

@@ -0,0 +1,7 @@
+require 'dnapi'
+base_dir = File.dirname(__FILE__)
+
+file_store_path   base_dir
+file_cache_path   base_dir
+cookbook_path     "#{base_dir}/cookbooks"
+node_name         DNApi.from(File.read("/etc/chef/dna.json")).id

data/htttee.gemspec

@@ -0,0 +1,30 @@
+Gem::Specification.new do |s|
+  s.name        = "htttee"
+  s.version     = '0.5.0'
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ben Burkert"]
+  s.email       = ["ben@benburkert.com"]
+  s.homepage    = "http://github.com/benburkert/htttee"
+  s.summary     = %q{Unix's tee as a service}
+  s.description = %q{Stream any CLI output as an HTTP chunked response.}
+
+  s.rubyforge_project = "htttee"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+
+  s.add_dependency 'rack-client', '~> 0.3.1.pre.i'
+  s.add_dependency 'trollop'
+
+  s.add_development_dependency 'sinatra'
+  s.add_development_dependency 'thin'
+  s.add_development_dependency 'em-redis'
+  s.add_development_dependency 'yajl-ruby'
+  s.add_development_dependency 'rack-mux'
+
+  s.add_development_dependency 'rspec'
+  s.add_development_dependency 'rake'
+  # s.add_development_dependency 'ruby-debug19'
+end

data/lib/htttee/client.rb

@@ -0,0 +1,16 @@
+require 'net/http'
+require 'rack/client'
+#require 'uuidtools'
+
+module EY
+  module Tea
+    module Client
+      def self.new(*a)
+        Consumer.new(*a)
+      end
+    end
+  end
+end
+
+require 'htttee/client/ext/net/http'
+require 'htttee/client/consumer'

data/lib/htttee/client/consumer.rb

@@ -0,0 +1,49 @@
+module EY
+  module Tea
+    module Client
+      class Consumer < Rack::Client::Base
+        def initialize(options = {})
+          @base_uri = URI.parse(options.fetch(:endpoint, 'http://tea.engineyard.com/'))
+          inner_app = options.fetch(:app, Rack::Client::Handler::NetHTTP.new)
+
+          super(inner_app)
+        end
+
+        def up(io, uuid, content_type = 'text/plain')
+          post("/#{uuid}", {'Content-Type' => content_type, 'Transfer-Encoding' => 'chunked'}, io)
+        end
+
+        def down(uuid)
+          get("/#{uuid}") do |status, headers, response_body|
+            response_body.each do |chunk|
+              yield chunk
+            end
+          end
+        end
+
+        def build_env(request_method, url, headers = {}, body = nil)
+          uri = @base_uri.nil? ? URI.parse(url) : @base_uri + url
+
+          env = super(request_method, uri.to_s, headers, body)
+
+          env['HTTP_HOST']       ||= http_host_for(uri)
+          env['HTTP_USER_AGENT'] ||= http_user_agent
+
+          env
+        end
+
+        def http_host_for(uri)
+          if uri.to_s.include?(":#{uri.port}")
+            [uri.host, uri.port].join(':')
+          else
+            uri.host
+          end
+        end
+
+        def http_user_agent
+          "htttee (rack-client #{Rack::Client::VERSION} (app: #{@app.class}))"
+        end
+      end
+    end
+  end
+end

data/lib/htttee/client/ext/net/http.rb

@@ -0,0 +1,27 @@
+module Net
+  class HTTP
+    class Post
+      def send_request_with_body_stream(sock, ver, path, f)
+        unless content_length() or chunked?
+          raise ArgumentError,
+            "Content-Length not given and Transfer-Encoding is not `chunked'"
+        end
+        supply_default_content_type
+        write_header sock, ver, path
+        if chunked?
+          begin
+            while s = f.readpartial(1024)
+              sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
+            end
+          rescue EOFError
+            sock.write "0\r\n\r\n"
+          end
+        else
+          while s = f.read(1024)
+            sock.write s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/htttee/server.rb

@@ -0,0 +1,74 @@
+require 'thin'
+require 'yajl'
+require 'sinatra/base'
+require 'em-redis'
+
+EM.epoll
+
+module EY
+  module Tea
+    module Server
+
+      def self.app
+        mocking? ? mock_app : rack_app
+      end
+
+      def self.api(host = (ENV['REDIS_HOST'] || 'localhost' ), port = (ENV['REDIS_PORT'] || 6379).to_i)
+        Api.new(host, port)
+      end
+
+      def self.rack_app
+        Rack::Builder.app do |builder|
+          builder.use AsyncFixer
+          builder.use Dechunker
+          builder.run Server.api
+        end
+      end
+
+      def self.mock_app
+        Rack::Builder.app do |builder|
+          builder.use Mock::ThinMuxer
+          builder.use Mock::EchoUri
+          builder.run Server.rack_app
+        end
+      end
+
+      def self.mock!
+        require 'htttee/server/mock'
+        @mocking = true
+
+        @mock_uri = Mock.boot_forking_server
+      end
+
+      def self.reset!
+        raise "Can't reset in non-mocked mode." unless mocking?
+
+        EM.run do
+          EM::Protocols::Redis.connect.flush_all do
+            EM.stop
+          end
+        end
+      end
+
+      def self.mocking?
+        @mocking
+      end
+
+      def self.mock_uri
+        raise "Not in mock mode!" unless mocking?
+        @mock_uri
+      end
+    end
+  end
+end
+
+require 'htttee/server/ext/em-redis'
+require 'htttee/server/ext/thin'
+
+require 'htttee/server/pubsub_redis'
+
+require 'htttee/server/api'
+require 'htttee/server/chunked_body'
+
+require 'htttee/server/middleware/async_fixer'
+require 'htttee/server/middleware/dechunker'

data/lib/htttee/server/api.rb

@@ -0,0 +1,201 @@
+module EY
+  module Tea
+    module Server
+      class Api
+        STREAMING, FIN = ?0, ?1
+        SUBSCRIBE, UNSUBSCRIBE, MESSAGE = 'SUBSCRIBE', 'UNSUBSCRIBE', 'MESSAGE'
+
+        AsyncResponse = [-1, {}, []].freeze
+
+        attr_accessor :redis
+
+        def initialize(host, port)
+          @host, @port = host, port
+        end
+
+        def call(env)
+          uuid = env['PATH_INFO'].sub(/^\//, '')
+
+          case env['REQUEST_METHOD']
+          when 'POST' then post(env, uuid)
+          when 'GET'  then get(env, uuid)
+          end
+
+          AsyncResponse
+        end
+
+        def post(env, uuid)
+          body = Thin::DeferrableBody.new
+
+          redis.set(state_key(uuid), STREAMING)
+
+          set_input_callback(env, uuid, body)
+          set_input_errback(env, uuid, body)
+          set_input_each(env, uuid, body)
+        end
+
+        def get(env, uuid)
+          body = ChunkedBody.new
+
+          with_state_for(uuid) do |state|
+            case state
+            when NilClass   then four_oh_four_response(env, body)
+            when STREAMING  then open_stream_response(env, uuid, body)
+            when FIN        then closed_stream_response(env, uuid, body)
+            end
+          end
+        end
+
+        def set_input_callback(env, uuid, body)
+          rack_input(env).callback do
+            async_callback(env).call [204, {}, body]
+
+            redis.set(state_key(uuid), FIN) do
+              finish(channel(uuid))
+              body.succeed
+            end
+          end
+        end
+
+        def set_input_errback(env, uuid, body)
+          rack_input(env).errback do |error|
+            async_callback(env).call [500, {}, body]
+
+            body.call [error.inspect]
+            body.succeed
+          end
+        end
+
+        def set_input_each(env, uuid, body)
+          rack_input(env).each do |chunk|
+            unless chunk.empty?
+              redis.pipeline(['append', data_key(uuid), chunk], ['publish', channel(uuid), Yajl::Encoder.encode([STREAMING, chunk])])
+            end
+          end
+        end
+
+        def four_oh_four_response(env, body)
+          env['async.callback'].call [404, {'Transfer-Encoding' => 'chunked'}, body]
+
+          body.succeed
+        end
+
+        def open_stream_response(env, uuid, body)
+          start_response(env, body)
+          stream_data_to(body, data_key(uuid)) do
+            with_state_for(uuid) do |state|
+              if state == FIN
+                body.succeed
+              else
+                subscribe_and_stream(env, uuid, body)
+              end
+            end
+          end
+        end
+
+        def closed_stream_response(env, uuid, body)
+          start_response(env, body)
+          stream_data_to(body, data_key(uuid)) do
+            body.succeed
+          end
+        end
+
+        def stream_data_to(body, key, offset = 0, chunk_size = 1024, &block)
+          redis.substr(key, offset, offset + chunk_size) do |chunk|
+            if chunk.nil? || chunk.empty?
+              yield
+            else
+              body.call [chunk]
+              stream_data_to(body, key, offset + chunk.size, chunk_size, &block)
+            end
+          end
+        end
+
+        def respond_with(env, body, data)
+          start_response(env, body, data)
+          body.succeed
+        end
+
+        def start_response(env, body, data = nil)
+          env['async.callback'].call [200, {'Transfer-Encoding' => 'chunked', 'Content-Type' => 'text/plain'}, body]
+
+          body.call [data] unless data.nil? || data.empty?
+        end
+
+        def subscribe_and_stream(env, uuid, body)
+          subscribe channel(uuid) do |type, message, *extra|
+            case type
+            #when SUBSCRIBE then start_response(env, body, data)
+            when FIN       then body.succeed
+            when STREAMING then body.call [message]
+            end
+          end
+        end
+
+        def with_state_for(uuid, &block)
+          redis.get(state_key(uuid), &block)
+        end
+
+        def with_state_and_data_for(uuid, &block)
+          redis.multi_get(state_key(uuid), data_key(uuid), &block)
+        end
+
+        def data_key(uuid)
+          "#{uuid}:data"
+        end
+
+        def state_key(uuid)
+          "#{uuid}:state"
+        end
+
+        def channel(uuid)
+          uuid
+        end
+
+        def rack_input(rack_env)
+          rack_env['rack.input']
+        end
+
+        def async_callback(rack_env)
+          rack_env['async.callback']
+        end
+
+        def publish(channel, data)
+          redis.publish channel, Yajl::Encoder.encode([STREAMING, data])
+        end
+
+        def finish(channel)
+          redis.publish channel, Yajl::Encoder.encode([FIN])
+        end
+
+        def subscribe(channel, &block)
+          conn = pubsub
+          conn.subscribe channel do |type, chan, data|
+            case type.upcase
+            when SUBSCRIBE then block.call(SUBSCRIBE, chan)
+            when MESSAGE
+              state, data = Yajl::Parser.parse(data)
+              case state
+              when STREAMING  then block.call(STREAMING, data)
+              when FIN
+                conn.unsubscribe channel
+                block.call(FIN, data)
+              end
+            else
+              debugger
+              ''
+            end
+          end
+        end
+
+        def pubsub
+          EM::Protocols::PubSubRedis.connect(@host, @port)
+        end
+
+        def redis
+          @redis ||= EM::Protocols::Redis.connect(@host, @port)
+        end
+      end
+    end
+  end
+end