htttee 0.5.0

data/.gitignore

@@ -0,0 +1,6 @@
+.redcar
+.DS_Store
+
+Gemfile.lock
+pkg
+vendor

data/Gemfile

@@ -0,0 +1,2 @@
+source 'http://rubygems.org/'
+gemspec

data/README.md

@@ -0,0 +1,105 @@
+# htttee - unix's tee-as-a-service
+
+
+## What is 'tee'?
+
+    $ man tee
+    NAME
+         tee -- pipe fitting
+    DESCRIPTION
+         The tee utility copies standard input to standard output, making a copy in zero or more files.  The output is unbuffered.
+
+It's very handy for piping the output of a script to a file and to STDOUT simultaneously.
+
+## What is 'htttee'?
+
+Instead of piping to a file, `htttee` pipes to a web service. Consumers can then stream the
+piped output via the `htttee` command line. Alternately, the streamed output could be viewed
+within a browser.
+
+That is, a streaming output locked within a server can be made accessible to a console
+or browser via `htttee`.
+
+## Usage
+
+In one terminal:
+
+    ruby -e "STDOUT.sync = true; 1.upto(100) {|i| puts i; sleep(i/100.0)}" | \
+      htttee -e http://localhost:3000 -u SOMEUNIQUESTRING
+
+In another terminal:
+
+    curl http://htttee.engineyard.com/SOMEUNIQUESTRING
+
+Or to see the chunked information:
+
+    $ telnet localhost 3000
+
+Then enter:
+
+    GET /1234 HTTP/1.1
+    Host: localhost
+
+You will then see a flow of integers being chunked preceded by the size of the chunks. Here is the 
+final few numbers chunked through:
+
+    3
+    98
+    
+    3
+    99
+    
+    4
+    100
+    
+    0
+    
+    Connection closed by foreign host.
+
+
+## Running the server
+
+    git clone git://github.com/benburkert/htttee.git
+    cd htttee
+    bundle
+    thin start
+
+## Development
+
+To pull down the repository and run the test suite:
+
+    git clone git://github.com/benburkert/htttee.git
+    cd htttee
+    bundle
+    rake
+
+To install the gem locally from source:
+
+    rake install
+
+To release the gem:
+
+    rake release
+
+## License
+
+Copyright (c) 2011 Ben Burkert, ben@benburkert.com
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/Rakefile

@@ -0,0 +1,14 @@
+require 'bundler'
+Bundler::GemHelper.install_tasks
+
+begin
+  require 'rspec/core/rake_task' rescue nil
+  RSpec::Core::RakeTask.new do |t|
+    t.rspec_opts = %w[ -b -c -f documentation -r ./spec/spec_helper.rb ]
+    t.pattern = 'spec/**/*_spec.rb'
+  end
+
+  task :default => :spec
+
+rescue LoadError
+end

data/bin/htttee

@@ -0,0 +1,39 @@
+#!/usr/bin/env ruby
+
+require 'trollop'
+require 'htttee/client'
+
+class MultiplexedIO < IO
+  instance_methods.each { |m| undef_method m unless (m =~ /^__/ || m =~ /^object_id$/ ) }
+
+  def initialize(input, output)
+    @input, @output = input, output
+  end
+
+  def readpartial(*a, &b)
+    data = @input.readpartial(*a,&b)
+    @output << data
+    data
+  end
+
+  def method_missing(method, *a, &b)
+    @input.send(method, *a, &b)
+  end
+end
+
+opts = Trollop::options do
+  opt :uuid, "The UUID of the stream.", :short => '-u', :type => String
+  opt :'content-type', "The content-type of the stream.", :short => '-c', :type => String, :default => 'text/plain'
+  opt :endpoint, "The endpoint of the htttee service.", :short => '-e', :type => String, :default => 'http://htttee.engineyard.com/'
+  opt :quiet, "Don't echo to stdout.", :short => '-q', :default => false
+end
+
+Trollop::die :uuid, "is required" if opts[:uuid].nil?
+
+$stdin.sync  = true
+$stdout.sync = true
+
+input  = opts[:quiet] ? $stdin : MultiplexedIO.new($stdin, $stdout)
+client = EY::Tea::Client.new(:endpoint => opts[:endpoint])
+
+client.up(input, opts[:uuid], opts[:'content-type'])

data/bin/htttee-exec

@@ -0,0 +1,18 @@
+#!/bin/sh
+
+stdout_uuid="$(head -c4096 /dev/urandom | sha256sum | awk '{ print $1 }')"
+stderr_uuid="$(head -c4096 /dev/urandom | sha256sum | awk '{ print $1 }')"
+
+mkfifo $stdout_uuid
+mkfifo $stderr_uuid
+
+bin/htttee --uuid $stdout_uuid --endpoint http://127.0.0.1:9292/ < $stdout_uuid &
+bin/htttee --uuid $stderr_uuid --endpoint http://127.0.0.1:9292/ < $stderr_uuid &
+
+echo "stdout: http://127.0.0.1:9292/$stdout_uuid"
+echo "stderr: http://127.0.0.1:9292/$stderr_uuid"
+
+exec 1>$stdout_uuid
+exec 2>$stderr_uuid
+
+exec $*

data/config.ru

@@ -0,0 +1,6 @@
+$:.unshift File.join(File.dirname(__FILE__), 'lib')
+
+require 'htttee/server'
+
+use Rack::CommonLogger
+run EY::Tea::Server.app

data/deploy/after_restart.rb

@@ -0,0 +1 @@
+sudo "god signal resque-#{app} QUIT"

data/deploy/before_restart.rb

@@ -0,0 +1,4 @@
+on_app_servers do
+  env_custom = "#{shared_path}/config/env.custom"
+  run "echo 'export APP_CURRENT_PATH=\"#{current_path}\"' > #{env_custom}"
+end

data/deploy/before_symlink.rb

@@ -0,0 +1,2 @@
+sudo "/usr/local/ey_resin/ruby/bin/chef-solo " \
+  "-c #{latest_release}/deploy/solo.rb -j /etc/chef/dna.json"

data/deploy/cookbooks/cloudkick-plugins/recipes/default.rb

@@ -0,0 +1,8 @@
+directory "/usr/lib/cloudkick-agent/plugins" do
+  owner "root"
+  group "root"
+  action :create
+  recursive true
+end
+
+require_recipe "cloudkick-plugins::resque"

data/deploy/cookbooks/cloudkick-plugins/recipes/resque.rb

@@ -0,0 +1,18 @@
+execute 'restart-cloudkick-agent' do
+  action :nothing
+
+  command 'pkill cloudkick-agent'
+end
+
+template "/usr/lib/cloudkick-agent/plugins/resque" do
+  mode "0755"
+  source "resque.sh.erb"
+  variables(
+    :dir       => "/data/#{node.engineyard.environment.apps.first.name}/current",
+    :script    => "script/cloudkick-resque",
+    :rack_env  => node.engineyard.environment.framework_env,
+    :redis_uri => "#{node.engineyard.environment.db_host}:6379"
+  )
+
+  notifies :run, resources(:execute => 'restart-cloudkick-agent')
+end

data/deploy/cookbooks/cloudkick-plugins/templates/default/resque.sh.erb

@@ -0,0 +1,4 @@
+#!/bin/sh
+export RACK_ENV=<%= @rack_env %>
+export REDIS_URI=<%= @redis_uri %>
+cd <%= @dir %> && bundle exec <%= @script %>

data/deploy/cookbooks/god/recipes/default.rb

@@ -0,0 +1,50 @@
+execute "restart god" do
+  command <<-SRC
+    if pgrep god; then
+      god quit
+    fi
+  SRC
+  action :nothing
+end
+
+gem_package "god" do
+  version "0.11.0"
+  action :install
+  notifies :run, resources(:execute => "restart god")
+end
+
+directory "/etc/god" do
+  owner 'root'
+  group 'root'
+  mode 0755
+  recursive true
+end
+
+template "/etc/god/config" do
+  owner "root"
+  group "root"
+  mode 0644
+  source "config.erb"
+end
+
+execute "telinit q" do
+  command "telinit q"
+  action :nothing
+end
+
+template "/tmp/god-inittab" do
+  owner "root"
+  group "root"
+  mode 0644
+  source "god-inittab.erb"
+end
+
+execute "make init work with god" do
+  command "cat /tmp/god-inittab >>/etc/inittab"
+  not_if "grep '# god config' /etc/inittab"
+  notifies :run, resources(:execute => "telinit q")
+end
+
+file "/tmp/god-inittab" do
+  action :delete
+end

data/deploy/cookbooks/god/templates/default/config.erb

@@ -0,0 +1,3 @@
+Dir.glob("/etc/god/*.rb").each do |filename|
+  load filename
+end

data/deploy/cookbooks/god/templates/default/god-inittab.erb

@@ -0,0 +1,3 @@
+# god config
+god:345:respawn:/usr/bin/god -c /etc/god/config -l /var/log/god.log --log-level info -D >> /root/god.log 2>&1
+god0:06:wait:/usr/bin/god quit

data/deploy/cookbooks/main/attributes/owner_name.rb

@@ -0,0 +1,3 @@
+# Set the owner_name to the first username
+owner_name(@attribute[:users].first[:username])
+owner_pass(@attribute[:users].first[:password])

data/deploy/cookbooks/main/attributes/recipes.rb

@@ -0,0 +1 @@
+recipes "main"

data/deploy/cookbooks/main/libraries/dnapi.rb

@@ -0,0 +1,7 @@
+require "dnapi"
+
+class Chef::Node
+  def engineyard
+    @engineyard ||= DNApi.from(File.read("/etc/chef/dna.json"))
+  end
+end

data/deploy/cookbooks/main/recipes/default.rb

@@ -0,0 +1,2 @@
+require_recipe "resque"
+#require_recipe "cloudkick-plugins"

data/deploy/cookbooks/nginx/recipes/default.rb

@@ -0,0 +1,2 @@
+require_recipe "nginx::install"
+require_recipe "nginx::configure"

data/deploy/cookbooks/nginx/templates/default/nginx.conf.erb

@@ -0,0 +1,41 @@
+user ey ey;
+worker_processes 4;
+pid /var/run/nginx.pid;
+
+events {
+  worker_connections 8192;
+  use epoll;
+}
+
+http {
+
+  upstream thins {
+    server unix:/tmp/thin.sock;
+  }
+
+  access_log  /var/log/foo.log;
+  error_log /var/bar.log debug;
+
+  server {
+    listen 80;
+    server_name *.amazonaws.com;
+
+    chunkin on;
+    proxy_buffering off;
+    chunked_transfer_encoding off;
+
+    location / {
+      proxy_pass        http://thins;
+      proxy_set_header  X-Real-IP  $remote_addr;
+      proxy_buffering off;
+      #chunked_transfer_encoding on;
+
+    }
+
+    error_page 411 = @my_411_error;
+    location @my_411_error {
+        proxy_buffering off;
+        chunkin_resume;
+    }
+  }
+}

data/deploy/cookbooks/resque/recipes/default.rb

@@ -0,0 +1,47 @@
+if node.engineyard.name == "resque" || node.engineyard.role == "solo"
+  require_recipe "god"
+
+  node[:applications].each do |app, data|
+    template "/etc/god/resque_#{app}.rb" do
+      owner "root"
+      group "root"
+      mode 0644
+      source "resque.rb.erb"
+      variables(
+        :app                  => app,
+        :app_root             => "/data/#{app}/current",
+        :resque_workers_count => 4
+      )
+      notifies :run, resources(:execute => "restart god")
+    end
+
+    template "/etc/god/resque_scheduler_#{app}.rb" do
+      owner "root"
+      group "root"
+      mode 0644
+      source "resque_scheduler.rb.erb"
+      variables(
+        :app                  => app,
+        :app_root             => "/data/#{app}/current",
+        :resque_workers_count => 1
+      )
+      notifies :run, resources(:execute => "restart god")
+    end
+  end
+end
+
+if %w[solo app app_master util].include? node[:instance_role]
+  node[:applications].each do |app, data|
+    template "/data/#{app}/shared/config/resque.yml" do
+      owner node[:owner_name]
+      group node[:owner_name]
+      mode 0655
+      source "resque.yml.erb"
+      variables(
+        :framework_env => node.engineyard.environment.framework_env,
+        :redis_host    => node.engineyard.environment.db_host,
+        :redis_port    => 6379
+      )
+    end
+  end
+end

data/deploy/cookbooks/resque/templates/default/resque.rb.erb

@@ -0,0 +1,73 @@
+rack_env   = "<%= @node[:environment][:framework_env] %>"
+app_name   = "<%= @app %>"
+app_root   = "<%= @app_root %>"
+owner      = "<%= @node[:owner_name] %>"
+home      = "/home/#{owner}"
+instance_id = "<%= @node.engineyard.id %>"
+
+<%= @resque_workers_count %>.times do |num|
+  inline = "#{home}/.ruby_inline/resque-#{app_name}-#{num}"
+
+  God.watch do |w|
+    w.name     = "resque-#{app_name}-#{num}"
+    w.group    = "resque-#{app_name}"
+    w.uid      = owner
+    w.gid      = owner
+    w.interval = 30.seconds
+    w.log      = "#{app_root}/log/worker.#{num}.log"
+    w.dir      = app_root
+    w.env      = {
+      "USER"        => owner,
+      "VERBOSE"     => "true",
+      "INSTANCE_ID" => instance_id,
+      "GOD_WATCH"   => w.name,
+      "RACK_ENV"    => rack_env,
+      "HOME"        => home,
+      "QUEUE"       => "*",
+    }
+
+    w.start    = "bundle exec rake --trace init resque:work"
+
+    w.behavior(:clean_pid_file)
+
+    w.start_grace   = 2.minutes
+    w.restart_grace = 2.minutes
+
+    # retart if memory gets too high
+    w.transition(:up, :restart) do |on|
+      on.condition(:memory_usage) do |c|
+        c.above = 350.megabytes
+        c.times = 2
+      end
+    end
+
+    # determine the state on startup
+    w.transition(:init, { true => :up, false => :start }) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+      end
+    end
+
+    # determine when process has finished starting
+    w.transition([:start, :restart], :up) do |on|
+      on.condition(:process_running) do |c|
+        c.running = true
+        c.interval = 5.seconds
+      end
+
+      # failsafe
+      on.condition(:tries) do |c|
+        c.times = 5
+        c.transition = :start
+        c.interval = 5.seconds
+      end
+    end
+
+    # start if process is not running
+    w.transition(:up, :start) do |on|
+      on.condition(:process_running) do |c|
+        c.running = false
+      end
+    end
+  end
+end