yarn 0.0.1 → 0.0.2

data/.gitignore

@@ -1,3 +1,4 @@
+tags
 coverage/*
 test_objects/rdoc/*
 *.gem

data/Gemfile

@@ -3,8 +3,5 @@ source "http://rubygems.org"
 # Specify your gem's dependencies in threaded_server.gemspec
 gemspec
 
-# gems for testing
 gem 'autotest'
-
-# development gems
 gem 'pry'

data/README.md

@@ -1,3 +1,27 @@
 # Yarn #
 
-A multi-threaded webserver written in Ruby 1.9 by Jesper Kjeldgaard.
+Yarn is a multi-threaded webserver written in Ruby 1.9 by Jesper Kjeldgaard.
+It handles concurrent requests by means of a set of workers and a job queue for incomming requests.
+
+Supports:
+* 
+
+
+## Installation ##
+`gem install yarn`
+
+
+## Usage ##
+To use Yarn with Rack applications:
+
+`rackup -s Yarn <rackup file (config.ru)>`
+
+
+To use Yarn for serving static and ruby (*.rb) files:
+
+
+    yarn [options]
+    where [options] are:
+      --host, -h <s>:   Hostname or IP address of the server (default: 127.0.0.1)
+      --port, -p <i>:   Port number to listen on for incomming requests (default: 3000)
+      --workers, -w <s>:   Number of worker threads (default: 32)

data/bin/yarn

@@ -7,9 +7,9 @@ require 'trollop'
 require 'yarn'
 
 opts = Trollop::options do
-  version "Yarn v.#{Yarn::VERSION} 2011 Jesper Kjeldgaard"
+  version "Yarn v#{Yarn::VERSION} 2011 Jesper Kjeldgaard"
   banner <<-EOS
-Yarn v.#{Yarn::VERSION} is a multithreaded webserver written in Ruby 1.9.
+Yarn v#{Yarn::VERSION} is a multiprocess webserver written in Ruby 1.9.2
 
 Usage: yarn [options]
 where [options] are:
@@ -17,8 +17,10 @@ EOS
 
   opt :host, "Hostname or IP address of the server", :default => "127.0.0.1"
   opt :port, "Port number to listen on for incomming requests", :default => 3000
-  opt :rackup_file, "Rackup file (e.g. config.ru). If not given Yarn will serve static and dynamic (*.rb) files.", :type => String
+  opt :workers, "Number of worker threads", :default => 4
+  opt :rack, "Rackup file <config.ru>", :default => "off"
+  opt :debug, "Output debug messages"
 end
 
-server = Yarn::Server.new(nil,opts)
+server = Yarn::Server.new(opts)
 server.start

data/features/concurrency.feature

@@ -2,14 +2,14 @@ Feature: Concurrency
 
   As a developer
   I want to be able to serve multiple requests in parallel
-  To increase server throughput
+  To increase server performance
 
   Background:
-    Given the server is running as "dynamic"
+    Given the server is running
 
   Scenario: Perform two requests in parallel
     Given a client "A"
     And a client "B"
-    When client "A" makes a "3" seconds request
-    And client "B" makes a "1" second request
+    When client "A" makes a "1" seconds request
+    And client "B" makes a "0.1" second request
     Then client "B" receives a response before client "A"

data/features/dynamic_request.feature

@@ -5,9 +5,14 @@ Feature: Dynamic request
   In order to provide dynamic content
 
   Background:
-    Given the server is running as "dynamic"
+    Given the server is running
 
   Scenario: Serve a dynamic Ruby file
     Given the file "/app.rb" exist
     When I go to "/app.rb"
     Then the response should contain "Dynamic request complete"
+
+  Scenario: Support POST data
+    Given the file "/post_app.rb" exist
+    When I post "field1" as "value1" to "/test_objects/post_app.rb"
+    Then the response should be "Recieved field1=value1"

data/features/rack.feature

@@ -4,15 +4,12 @@ Feature: Implement rack interface
   I want to have a rack handler
   In order to serve rack applications
 
-  @wip
   Scenario: Serve a one-file rack application
-    Given I have a rack application "simple_rack.rb"
-    And the server is running as "rack"
+    Given the rack test app is running
     When I go to "/"
-    Then the response should contain "rack works"
+    Then the response should contain "Rack works"
 
   Scenario: Serve a rails application
-    Given I have a rails application "rails_test"
-    And the server is running as "rack"
-    When I go to "/rails_text/home/index"
-    Then the response should contain "Rack rails works"
+    Given the rails test app is running
+    When I go to ""
+    Then the response should contain "Yarn Test Blog"

data/features/static_request.feature

@@ -5,7 +5,7 @@ Feature: Static file requests
   To provide fast content on the Internet
 
   Background:
-    Given the server is running as "static"
+    Given the server is running
 
   Scenario: Serve a static html file
     Given the file "index.html" exist
@@ -22,4 +22,4 @@ Feature: Static file requests
     Given the file "non-existent-file.html" does not exist
     When I go to "non-existent-file.html"
     Then the response should contain "404"
-    Then the response should contain "File does not exist"
+    Then the response should contain "does not exist"

data/features/step_definitions/rack_steps.rb

@@ -1,3 +1,16 @@
+require 'rack'
+
 Given /^I have a rack application "([^"]*)"$/ do |app|
    testfile_exists?(app).should be_true 
 end
+
+Given /^the rack test app is running$/ do
+  start_server(3000,"test_objects/config.ru")
+end
+
+Given /^the rails test app is running$/ do
+  current_dir = Dir.pwd
+  Dir.chdir("test_objects/rails_test")
+  start_server(3000,"config.ru")
+  Dir.chdir(current_dir)
+end

data/features/step_definitions/server_steps.rb

@@ -9,11 +9,7 @@ When /^I stop the server$/ do
 end
 
 Given /^the server is running$/ do
-  start_server
-end
-
-Given /^the server is running as "([^"]*)"$/ do |handler_type|
-  start_server(3000, handler_type.to_sym)
+  start_server(3000)
 end
 
 Given /^the server is not running$/ do
@@ -40,3 +36,7 @@ end
 Then /^I should see "([^"]*)"$/ do |message|
   $console.contains? message
 end
+
+Then /^Pry$/ do
+  binding.pry
+end

data/features/step_definitions/web_steps.rb

@@ -5,3 +5,11 @@ end
 Then /^the response should contain "([^"]*)"$/ do |content|
   @response.body.should include(content)
 end
+
+Then /^the response should be "([^"]*)"$/ do |content|
+  @response.body.gsub(/\n?/,"").should == content
+end
+
+When /^I post "([^"]*)" as "([^"]*)" to "([^"]*)"$/ do |key, value, url|
+  @response = post(url, { key.to_sym => value })
+end

data/lib/rack/handler/yarn.rb

@@ -6,14 +6,15 @@ module Rack
   module Handler
     class Yarn
       def self.run(app, options={})
-        server = ::Yarn::Server.new(app,options)
-        server.start
+        options = options.merge({ rack: app })
+        @server = ::Yarn::Server.new(options)
+        @server.start
       end
 
       def self.valid_options
         {
           "Host=HOST" => "Hostname to listen on (default: 127.0.0.1)",
-          "Port=PORT" => "Port to listen on (default: 3000)",
+          "Port=PORT" => "Port to listen on (default: 3000)"
         }
       end
     end

data/lib/yarn/error_page.rb

@@ -4,7 +4,8 @@ module Yarn
 
     def serve_404_page
       @response.status = 404
-      @response.body = ["<html><head><title>404</title></head><body><h1>File does not exist.</h1></body><html>"]
+      fn = @request[:uri][:path] if @request
+      @response.body = ["<html><head><title>404</title></head><body><h1>File #{fn} does not exist.</h1></body><html>"]
     end
 
     def serve_500_page

data/lib/yarn/logging.rb

@@ -15,7 +15,7 @@ module Yarn
     end
 
     def debug(msg=nil)
-      log "DEBUG: #{msg || yield}"
+      log "DEBUG: #{msg}" if $debug
     end
 
     def output

data/lib/yarn/parslet_parser.rb

@@ -15,7 +15,6 @@ module Yarn
     rule(:spaces) { match('\s+') }
 
     # header rules
-
     rule(:header_value) { match['^\r\n'].once }
 
     rule(:header_name) { match['a-zA-Z\-'].once }
@@ -27,28 +26,18 @@ module Yarn
       header_value.as(:value).maybe >> 
       crlf.maybe 
     end
-    
-    # request-line rules
 
+    # request-line rules
     rule(:http_version) { match['HTTP\/\d\.\d'].once }
 
-    rule(:param_value) { match['^&\s+'].once }
-
-    rule(:param_name) { match['^=+'].once }
-
-    rule(:param) do
-      param_name.as(:name) >>
-      str("=") >>
-      param_value.as(:value) >>
-      str("&").maybe
-    end
-
     rule(:query) do
       match['\S+'].repeat(1)
     end
 
     rule(:path) do 
-      match['^\?'].repeat(1).as(:path) >> str("?") >> query.as(:query) | match['^\s'].once.as(:path)
+      match['^\?'].repeat(1).as(:path) >> 
+      str("?") >> 
+      query.as(:query) | match['^\s'].once.as(:path)
     end
 
     rule(:port) { match['\d+'].repeat(1) }
@@ -78,11 +67,16 @@ module Yarn
       http_version.as(:version) >> 
       crlf.maybe 
     end
+     
+    # body rule
+    rule(:body) { match['\S'].once }
 
     # RFC2616: Request-Line *(( header ) CRLF) CRLF [ message-body ]
     rule(:request) do 
       request_line >> 
       header.repeat.as(:_process_headers).as(:headers) >> 
+      crlf.maybe >>
+      body.as(:body).maybe >>
       crlf.maybe
     end
 
@@ -91,19 +85,10 @@ module Yarn
 
     def run(input)
       tree = parse input 
-      tree = ParamsTransformer.new.apply tree 
       HeadersTransformer.new.apply tree 
     end
   end
 
-  class ParamsTransformer < Parslet::Transform
-    rule(:_process_params => subtree(:params)) do
-      hash = {}
-      params.each { |h| hash[h[:name].to_s] = h[:value] }
-      hash
-    end
-  end
-
   class HeadersTransformer < Parslet::Transform
     rule(:_process_headers => subtree(:headers)) do
       hash = {}

data/lib/yarn/rack_handler.rb

@@ -4,16 +4,18 @@ require 'pry'
 module Yarn
   class RackHandler < RequestHandler
 
-    def initialize(app, opts)
-      super(opts)
-      @host,@port = opts[:host], opts[:port]
+    attr_accessor :env
+
+    def initialize(app)
+      @parser = ParsletParser.new
+      @response = Response.new
       @app = app
     end
 
     def prepare_response
       begin
-        env = make_env
-        @response.content = @app.call(env)
+        make_env
+        @response.content = @app.call(@env)
       rescue Exception => e
         log e.message
         log e.backtrace
@@ -21,18 +23,18 @@ module Yarn
     end
 
     def make_env
-      env = {
+      @env = {
         "REQUEST_METHOD"    => @request[:method].to_s,
         "PATH_INFO"         => @request[:uri][:path].to_s,
         "QUERY_STRING"      => @request[:uri][:query].to_s,
-        "SERVER_NAME"       => @host || @request[:uri][:host].to_s,
-        "SERVER_PORT"       => @port.to_s || @request[:uri][:port].to_s,
+        "SERVER_NAME"       => @request[:uri][:host].to_s,
+        "SERVER_PORT"       => @request[:uri][:port].to_s,
         "SCRIPT_NAME"       => "",
         "rack.input"        => StringIO.new("").set_encoding(Encoding::ASCII_8BIT),
         "rack.version"      => Rack::VERSION,
         "rack.errors"       => $output,
         "rack.multithread"  => true,
-        "rack.multiprocess" => false,
+        "rack.multiprocess" => true,
         "rack.run_once"     => false,
         "rack.url_scheme"   => "http"
       }

data/lib/yarn/request_handler.rb

@@ -14,24 +14,26 @@ module Yarn
 
     attr_accessor :session, :parser, :request, :response
 
-    def initialize(options={})
+    def initialize
       @parser = ParsletParser.new
       @response = Response.new
     end
 
     def run(session)
-      @response = Response.new
       set_common_headers
       @session = session
       begin
         parse_request
+        debug "Request parsed, path: #{request_path}"
         prepare_response
+        debug "Response prepared: #{@response.status}"
         return_response
-        log "Served (#{STATUS_CODES[@response.status]}) #{client_address} #{request_path}"
+        log "#{STATUS_CODES[@response.status]} #{client_address} #{request_path}"
       rescue EmptyRequestError
         log "Empty request from #{client_address}"
       ensure
         close_connection
+        debug "Connection closed"
       end
     end
 
@@ -72,7 +74,7 @@ module Yarn
     end
 
     def execute_script(path)
-      response = `ruby #{path}`
+      response = `ruby #{path} #{post_body}`
       if !! ($?.to_s =~ /1$/)
         raise ProcessingError
       else
@@ -80,6 +82,10 @@ module Yarn
       end
     end
 
+    def post_body
+      @request ? @request[:body].to_s : ""
+    end
+
     def return_response
       @session.puts "HTTP/1.1 #{@response.status} #{STATUS_CODES[@response.status]}"
       @session.puts @response.headers.map { |k,v| "#{k}: #{v}" }
@@ -101,9 +107,17 @@ module Yarn
     def read_request
       input = []
       while (line = @session.gets) do
-        break if line.length <= 2
-        input << line
+        length = line.gsub(/\D/,"") if line =~ /Content-Length/
+        if line == "\r\n"
+          input << line
+          input << @session.read(length.to_i) if length
+          break
+        else
+          input << line
+        end
       end
+      @session.close_read
+      debug "Done reading request"
       input.join
     end
 
@@ -141,12 +155,7 @@ module Yarn
 
     def read_file(path)
       file_contents = []
-
-      File.open(path, "r") do |file|
-        while (line = file.gets) do
-          file_contents << line
-        end
-      end
+      File.open(path).each { |line| file_contents << line }
 
       file_contents
     end

data/lib/yarn/server.rb

@@ -5,55 +5,64 @@ module Yarn
 
     include Logging
 
-    attr_accessor :host, :port, :socket, :socket_listener
+    attr_accessor :host, :port, :socket, :workers
 
-    def initialize(app=nil,opts={})
+    def initialize(options={})
       # merge given options with default values
-      options = { 
+      opts = { 
         output: $stdout, 
         host: '127.0.0.1', 
-        port: 3000 
-      }.merge(opts)
+        port: 3000,
+        workers: 4,
+        rack: "off" 
+      }.merge(options)
 
-      @app = app
-      @host,@port,$output = options[:host], options[:port], options[:output]
+      @app = nil
+      @app = load_rack_app(opts[:rack]) unless opts[:rack] == "off"
 
+      @host, @port, @num_workers = opts[:host], opts[:port], opts[:workers]
+      @workers = []
+      $output, $debug = opts[:output], opts[:debug]
+    end
+
+    def load_rack_app(app_path)
+      if File.exists?(app_path)
+        config_file = File.read(app_path)
+        rack_application = eval("Rack::Builder.new { #{config_file} }")
+      else
+        log "#{app_path} does not exist. Exiting."
+        Kernel::exit
+      end
+    end
+
+    def start
+      trap("INT") { stop }
       @socket = TCPServer.new(@host, @port)
+      log "Yarn started #{@num_workers} workers and is listening on #{@host}:#{@port}"
 
-      @handler = @app ? RackHandler.new(@app, options) : RequestHandler.new(options)
+      init_workers
 
-      log "Yarn started #{"w/ Rack " if opts[:rackup_file]}and accepting requests on #{@host}:#{@port}"
+      # Waits here for all processes to exit
+      Process.waitall
     end
 
-    def start
-      @socket_listener = Thread.new do
-        loop do
-          begin
+    def init_workers
+      @num_workers.times do
+        @workers << fork do
+          trap("INT") { exit }
+          loop do
+            handler ||= @app ? RackHandler.new(@app) : RequestHandler.new
             session = @socket.accept
-            Thread.new { @handler.clone.run session }
-          rescue Exception => e
-            session.close
-            log e.message
-            log e.backtrace
+            handler.run session 
           end
         end
       end
-
-      begin
-        @socket_listener.join
-      rescue Interrupt => e
-        log "Caught interrupt, stopping..."
-      ensure
-        stop
-      end
     end
 
     def stop
-      @socket.close if @socket
-      @socket = nil
-      @socket_listener.kill if @socket_listener
+      @socket.close if (@socket && !@socket.closed?)
 
-      log "Server stopped"
+      log "Server stopped. Have a nice day!"
     end
   end
 end