yarn 0.0.1 → 0.0.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (74) hide show
  1. data/.gitignore +1 -0
  2. data/Gemfile +0 -3
  3. data/README.md +25 -1
  4. data/bin/yarn +6 -4
  5. data/features/concurrency.feature +4 -4
  6. data/features/dynamic_request.feature +6 -1
  7. data/features/rack.feature +5 -8
  8. data/features/static_request.feature +2 -2
  9. data/features/step_definitions/rack_steps.rb +13 -0
  10. data/features/step_definitions/server_steps.rb +5 -5
  11. data/features/step_definitions/web_steps.rb +8 -0
  12. data/lib/rack/handler/yarn.rb +4 -3
  13. data/lib/yarn/error_page.rb +2 -1
  14. data/lib/yarn/logging.rb +1 -1
  15. data/lib/yarn/parslet_parser.rb +9 -24
  16. data/lib/yarn/rack_handler.rb +11 -9
  17. data/lib/yarn/request_handler.rb +21 -12
  18. data/lib/yarn/server.rb +39 -30
  19. data/lib/yarn/version.rb +1 -1
  20. data/spec/helpers.rb +16 -8
  21. data/spec/rack/handler/yarn_spec.rb +21 -0
  22. data/spec/spec_helper.rb +1 -4
  23. data/spec/yarn/directory_lister_spec.rb +0 -5
  24. data/spec/yarn/logging_spec.rb +3 -2
  25. data/spec/yarn/parslet_parser_spec.rb +26 -0
  26. data/spec/yarn/rack_handler_spec.rb +15 -4
  27. data/spec/yarn/request_handler_spec.rb +17 -7
  28. data/spec/yarn/server_spec.rb +49 -8
  29. data/test_objects/.gitignore +1 -0
  30. data/test_objects/app.rb +0 -2
  31. data/test_objects/config.ru +1 -49
  32. data/test_objects/rails_test/.gitignore +5 -0
  33. data/test_objects/rails_test/Gemfile +34 -0
  34. data/test_objects/rails_test/README +261 -0
  35. data/test_objects/rails_test/Rakefile +7 -0
  36. data/test_objects/rails_test/app/assets/images/rails.png +0 -0
  37. data/test_objects/rails_test/app/assets/javascripts/application.js +6 -0
  38. data/test_objects/rails_test/app/assets/stylesheets/application.css +7 -0
  39. data/test_objects/rails_test/app/assets/stylesheets/scaffolds.css.scss +56 -0
  40. data/test_objects/rails_test/app/mailers/.gitkeep +0 -0
  41. data/test_objects/rails_test/app/models/.gitkeep +0 -0
  42. data/test_objects/rails_test/app/views/layouts/application.html.erb +15 -0
  43. data/test_objects/rails_test/app/views/posts/_form.html.erb +25 -0
  44. data/test_objects/rails_test/app/views/posts/edit.html.erb +6 -0
  45. data/test_objects/rails_test/app/views/posts/index.html.erb +20 -0
  46. data/test_objects/rails_test/app/views/posts/new.html.erb +5 -0
  47. data/test_objects/rails_test/app/views/posts/show.html.erb +15 -0
  48. data/test_objects/rails_test/config.ru +4 -0
  49. data/test_objects/rails_test/config/database.yml +25 -0
  50. data/test_objects/rails_test/config/locales/en.yml +5 -0
  51. data/test_objects/rails_test/doc/README_FOR_APP +2 -0
  52. data/test_objects/rails_test/lib/assets/.gitkeep +0 -0
  53. data/test_objects/rails_test/lib/tasks/.gitkeep +0 -0
  54. data/test_objects/rails_test/log/.gitkeep +0 -0
  55. data/test_objects/rails_test/public/404.html +26 -0
  56. data/test_objects/rails_test/public/422.html +26 -0
  57. data/test_objects/rails_test/public/500.html +26 -0
  58. data/test_objects/rails_test/public/favicon.ico +0 -0
  59. data/test_objects/rails_test/public/robots.txt +5 -0
  60. data/test_objects/rails_test/script/rails +6 -0
  61. data/test_objects/rails_test/test/fixtures/.gitkeep +0 -0
  62. data/test_objects/rails_test/test/fixtures/posts.yml +9 -0
  63. data/test_objects/rails_test/test/functional/.gitkeep +0 -0
  64. data/test_objects/rails_test/test/integration/.gitkeep +0 -0
  65. data/test_objects/rails_test/test/unit/.gitkeep +0 -0
  66. data/test_objects/rails_test/vendor/assets/stylesheets/.gitkeep +0 -0
  67. data/test_objects/rails_test/vendor/plugins/.gitkeep +0 -0
  68. data/yarn.gemspec +4 -8
  69. metadata +59 -59
  70. data/lib/yarn/worker.rb +0 -19
  71. data/lib/yarn/worker_pool.rb +0 -36
  72. data/spec/yarn/worker_pool_spec.rb +0 -23
  73. data/spec/yarn/worker_spec.rb +0 -26
  74. data/test_objects/simple_rack.rb +0 -12
data/.gitignore CHANGED
@@ -1,3 +1,4 @@
1
+ tags
1
2
  coverage/*
2
3
  test_objects/rdoc/*
3
4
  *.gem
data/Gemfile CHANGED
@@ -3,8 +3,5 @@ source "http://rubygems.org"
3
3
  # Specify your gem's dependencies in threaded_server.gemspec
4
4
  gemspec
5
5
 
6
- # gems for testing
7
6
  gem 'autotest'
8
-
9
- # development gems
10
7
  gem 'pry'
data/README.md CHANGED
@@ -1,3 +1,27 @@
1
1
  # Yarn #
2
2
 
3
- A multi-threaded webserver written in Ruby 1.9 by Jesper Kjeldgaard.
3
+ Yarn is a multi-threaded webserver written in Ruby 1.9 by Jesper Kjeldgaard.
4
+ It handles concurrent requests by means of a set of workers and a job queue for incomming requests.
5
+
6
+ Supports:
7
+ *
8
+
9
+
10
+ ## Installation ##
11
+ `gem install yarn`
12
+
13
+
14
+ ## Usage ##
15
+ To use Yarn with Rack applications:
16
+
17
+ `rackup -s Yarn <rackup file (config.ru)>`
18
+
19
+
20
+ To use Yarn for serving static and ruby (*.rb) files:
21
+
22
+
23
+ yarn [options]
24
+ where [options] are:
25
+ --host, -h <s>: Hostname or IP address of the server (default: 127.0.0.1)
26
+ --port, -p <i>: Port number to listen on for incomming requests (default: 3000)
27
+ --workers, -w <s>: Number of worker threads (default: 32)
data/bin/yarn CHANGED
@@ -7,9 +7,9 @@ require 'trollop'
7
7
  require 'yarn'
8
8
 
9
9
  opts = Trollop::options do
10
- version "Yarn v.#{Yarn::VERSION} 2011 Jesper Kjeldgaard"
10
+ version "Yarn v#{Yarn::VERSION} 2011 Jesper Kjeldgaard"
11
11
  banner <<-EOS
12
- Yarn v.#{Yarn::VERSION} is a multithreaded webserver written in Ruby 1.9.
12
+ Yarn v#{Yarn::VERSION} is a multiprocess webserver written in Ruby 1.9.2
13
13
 
14
14
  Usage: yarn [options]
15
15
  where [options] are:
@@ -17,8 +17,10 @@ EOS
17
17
 
18
18
  opt :host, "Hostname or IP address of the server", :default => "127.0.0.1"
19
19
  opt :port, "Port number to listen on for incomming requests", :default => 3000
20
- opt :rackup_file, "Rackup file (e.g. config.ru). If not given Yarn will serve static and dynamic (*.rb) files.", :type => String
20
+ opt :workers, "Number of worker threads", :default => 4
21
+ opt :rack, "Rackup file <config.ru>", :default => "off"
22
+ opt :debug, "Output debug messages"
21
23
  end
22
24
 
23
- server = Yarn::Server.new(nil,opts)
25
+ server = Yarn::Server.new(opts)
24
26
  server.start
@@ -2,14 +2,14 @@ Feature: Concurrency
2
2
 
3
3
  As a developer
4
4
  I want to be able to serve multiple requests in parallel
5
- To increase server throughput
5
+ To increase server performance
6
6
 
7
7
  Background:
8
- Given the server is running as "dynamic"
8
+ Given the server is running
9
9
 
10
10
  Scenario: Perform two requests in parallel
11
11
  Given a client "A"
12
12
  And a client "B"
13
- When client "A" makes a "3" seconds request
14
- And client "B" makes a "1" second request
13
+ When client "A" makes a "1" seconds request
14
+ And client "B" makes a "0.1" second request
15
15
  Then client "B" receives a response before client "A"
@@ -5,9 +5,14 @@ Feature: Dynamic request
5
5
  In order to provide dynamic content
6
6
 
7
7
  Background:
8
- Given the server is running as "dynamic"
8
+ Given the server is running
9
9
 
10
10
  Scenario: Serve a dynamic Ruby file
11
11
  Given the file "/app.rb" exist
12
12
  When I go to "/app.rb"
13
13
  Then the response should contain "Dynamic request complete"
14
+
15
+ Scenario: Support POST data
16
+ Given the file "/post_app.rb" exist
17
+ When I post "field1" as "value1" to "/test_objects/post_app.rb"
18
+ Then the response should be "Recieved field1=value1"
@@ -4,15 +4,12 @@ Feature: Implement rack interface
4
4
  I want to have a rack handler
5
5
  In order to serve rack applications
6
6
 
7
- @wip
8
7
  Scenario: Serve a one-file rack application
9
- Given I have a rack application "simple_rack.rb"
10
- And the server is running as "rack"
8
+ Given the rack test app is running
11
9
  When I go to "/"
12
- Then the response should contain "rack works"
10
+ Then the response should contain "Rack works"
13
11
 
14
12
  Scenario: Serve a rails application
15
- Given I have a rails application "rails_test"
16
- And the server is running as "rack"
17
- When I go to "/rails_text/home/index"
18
- Then the response should contain "Rack rails works"
13
+ Given the rails test app is running
14
+ When I go to ""
15
+ Then the response should contain "Yarn Test Blog"
@@ -5,7 +5,7 @@ Feature: Static file requests
5
5
  To provide fast content on the Internet
6
6
 
7
7
  Background:
8
- Given the server is running as "static"
8
+ Given the server is running
9
9
 
10
10
  Scenario: Serve a static html file
11
11
  Given the file "index.html" exist
@@ -22,4 +22,4 @@ Feature: Static file requests
22
22
  Given the file "non-existent-file.html" does not exist
23
23
  When I go to "non-existent-file.html"
24
24
  Then the response should contain "404"
25
- Then the response should contain "File does not exist"
25
+ Then the response should contain "does not exist"
@@ -1,3 +1,16 @@
1
+ require 'rack'
2
+
1
3
  Given /^I have a rack application "([^"]*)"$/ do |app|
2
4
  testfile_exists?(app).should be_true
3
5
  end
6
+
7
+ Given /^the rack test app is running$/ do
8
+ start_server(3000,"test_objects/config.ru")
9
+ end
10
+
11
+ Given /^the rails test app is running$/ do
12
+ current_dir = Dir.pwd
13
+ Dir.chdir("test_objects/rails_test")
14
+ start_server(3000,"config.ru")
15
+ Dir.chdir(current_dir)
16
+ end
@@ -9,11 +9,7 @@ When /^I stop the server$/ do
9
9
  end
10
10
 
11
11
  Given /^the server is running$/ do
12
- start_server
13
- end
14
-
15
- Given /^the server is running as "([^"]*)"$/ do |handler_type|
16
- start_server(3000, handler_type.to_sym)
12
+ start_server(3000)
17
13
  end
18
14
 
19
15
  Given /^the server is not running$/ do
@@ -40,3 +36,7 @@ end
40
36
  Then /^I should see "([^"]*)"$/ do |message|
41
37
  $console.contains? message
42
38
  end
39
+
40
+ Then /^Pry$/ do
41
+ binding.pry
42
+ end
@@ -5,3 +5,11 @@ end
5
5
  Then /^the response should contain "([^"]*)"$/ do |content|
6
6
  @response.body.should include(content)
7
7
  end
8
+
9
+ Then /^the response should be "([^"]*)"$/ do |content|
10
+ @response.body.gsub(/\n?/,"").should == content
11
+ end
12
+
13
+ When /^I post "([^"]*)" as "([^"]*)" to "([^"]*)"$/ do |key, value, url|
14
+ @response = post(url, { key.to_sym => value })
15
+ end
@@ -6,14 +6,15 @@ module Rack
6
6
  module Handler
7
7
  class Yarn
8
8
  def self.run(app, options={})
9
- server = ::Yarn::Server.new(app,options)
10
- server.start
9
+ options = options.merge({ rack: app })
10
+ @server = ::Yarn::Server.new(options)
11
+ @server.start
11
12
  end
12
13
 
13
14
  def self.valid_options
14
15
  {
15
16
  "Host=HOST" => "Hostname to listen on (default: 127.0.0.1)",
16
- "Port=PORT" => "Port to listen on (default: 3000)",
17
+ "Port=PORT" => "Port to listen on (default: 3000)"
17
18
  }
18
19
  end
19
20
  end
@@ -4,7 +4,8 @@ module Yarn
4
4
 
5
5
  def serve_404_page
6
6
  @response.status = 404
7
- @response.body = ["<html><head><title>404</title></head><body><h1>File does not exist.</h1></body><html>"]
7
+ fn = @request[:uri][:path] if @request
8
+ @response.body = ["<html><head><title>404</title></head><body><h1>File #{fn} does not exist.</h1></body><html>"]
8
9
  end
9
10
 
10
11
  def serve_500_page
@@ -15,7 +15,7 @@ module Yarn
15
15
  end
16
16
 
17
17
  def debug(msg=nil)
18
- log "DEBUG: #{msg || yield}"
18
+ log "DEBUG: #{msg}" if $debug
19
19
  end
20
20
 
21
21
  def output
@@ -15,7 +15,6 @@ module Yarn
15
15
  rule(:spaces) { match('\s+') }
16
16
 
17
17
  # header rules
18
-
19
18
  rule(:header_value) { match['^\r\n'].once }
20
19
 
21
20
  rule(:header_name) { match['a-zA-Z\-'].once }
@@ -27,28 +26,18 @@ module Yarn
27
26
  header_value.as(:value).maybe >>
28
27
  crlf.maybe
29
28
  end
30
-
31
- # request-line rules
32
29
 
30
+ # request-line rules
33
31
  rule(:http_version) { match['HTTP\/\d\.\d'].once }
34
32
 
35
- rule(:param_value) { match['^&\s+'].once }
36
-
37
- rule(:param_name) { match['^=+'].once }
38
-
39
- rule(:param) do
40
- param_name.as(:name) >>
41
- str("=") >>
42
- param_value.as(:value) >>
43
- str("&").maybe
44
- end
45
-
46
33
  rule(:query) do
47
34
  match['\S+'].repeat(1)
48
35
  end
49
36
 
50
37
  rule(:path) do
51
- match['^\?'].repeat(1).as(:path) >> str("?") >> query.as(:query) | match['^\s'].once.as(:path)
38
+ match['^\?'].repeat(1).as(:path) >>
39
+ str("?") >>
40
+ query.as(:query) | match['^\s'].once.as(:path)
52
41
  end
53
42
 
54
43
  rule(:port) { match['\d+'].repeat(1) }
@@ -78,11 +67,16 @@ module Yarn
78
67
  http_version.as(:version) >>
79
68
  crlf.maybe
80
69
  end
70
+
71
+ # body rule
72
+ rule(:body) { match['\S'].once }
81
73
 
82
74
  # RFC2616: Request-Line *(( header ) CRLF) CRLF [ message-body ]
83
75
  rule(:request) do
84
76
  request_line >>
85
77
  header.repeat.as(:_process_headers).as(:headers) >>
78
+ crlf.maybe >>
79
+ body.as(:body).maybe >>
86
80
  crlf.maybe
87
81
  end
88
82
 
@@ -91,19 +85,10 @@ module Yarn
91
85
 
92
86
  def run(input)
93
87
  tree = parse input
94
- tree = ParamsTransformer.new.apply tree
95
88
  HeadersTransformer.new.apply tree
96
89
  end
97
90
  end
98
91
 
99
- class ParamsTransformer < Parslet::Transform
100
- rule(:_process_params => subtree(:params)) do
101
- hash = {}
102
- params.each { |h| hash[h[:name].to_s] = h[:value] }
103
- hash
104
- end
105
- end
106
-
107
92
  class HeadersTransformer < Parslet::Transform
108
93
  rule(:_process_headers => subtree(:headers)) do
109
94
  hash = {}
@@ -4,16 +4,18 @@ require 'pry'
4
4
  module Yarn
5
5
  class RackHandler < RequestHandler
6
6
 
7
- def initialize(app, opts)
8
- super(opts)
9
- @host,@port = opts[:host], opts[:port]
7
+ attr_accessor :env
8
+
9
+ def initialize(app)
10
+ @parser = ParsletParser.new
11
+ @response = Response.new
10
12
  @app = app
11
13
  end
12
14
 
13
15
  def prepare_response
14
16
  begin
15
- env = make_env
16
- @response.content = @app.call(env)
17
+ make_env
18
+ @response.content = @app.call(@env)
17
19
  rescue Exception => e
18
20
  log e.message
19
21
  log e.backtrace
@@ -21,18 +23,18 @@ module Yarn
21
23
  end
22
24
 
23
25
  def make_env
24
- env = {
26
+ @env = {
25
27
  "REQUEST_METHOD" => @request[:method].to_s,
26
28
  "PATH_INFO" => @request[:uri][:path].to_s,
27
29
  "QUERY_STRING" => @request[:uri][:query].to_s,
28
- "SERVER_NAME" => @host || @request[:uri][:host].to_s,
29
- "SERVER_PORT" => @port.to_s || @request[:uri][:port].to_s,
30
+ "SERVER_NAME" => @request[:uri][:host].to_s,
31
+ "SERVER_PORT" => @request[:uri][:port].to_s,
30
32
  "SCRIPT_NAME" => "",
31
33
  "rack.input" => StringIO.new("").set_encoding(Encoding::ASCII_8BIT),
32
34
  "rack.version" => Rack::VERSION,
33
35
  "rack.errors" => $output,
34
36
  "rack.multithread" => true,
35
- "rack.multiprocess" => false,
37
+ "rack.multiprocess" => true,
36
38
  "rack.run_once" => false,
37
39
  "rack.url_scheme" => "http"
38
40
  }
@@ -14,24 +14,26 @@ module Yarn
14
14
 
15
15
  attr_accessor :session, :parser, :request, :response
16
16
 
17
- def initialize(options={})
17
+ def initialize
18
18
  @parser = ParsletParser.new
19
19
  @response = Response.new
20
20
  end
21
21
 
22
22
  def run(session)
23
- @response = Response.new
24
23
  set_common_headers
25
24
  @session = session
26
25
  begin
27
26
  parse_request
27
+ debug "Request parsed, path: #{request_path}"
28
28
  prepare_response
29
+ debug "Response prepared: #{@response.status}"
29
30
  return_response
30
- log "Served (#{STATUS_CODES[@response.status]}) #{client_address} #{request_path}"
31
+ log "#{STATUS_CODES[@response.status]} #{client_address} #{request_path}"
31
32
  rescue EmptyRequestError
32
33
  log "Empty request from #{client_address}"
33
34
  ensure
34
35
  close_connection
36
+ debug "Connection closed"
35
37
  end
36
38
  end
37
39
 
@@ -72,7 +74,7 @@ module Yarn
72
74
  end
73
75
 
74
76
  def execute_script(path)
75
- response = `ruby #{path}`
77
+ response = `ruby #{path} #{post_body}`
76
78
  if !! ($?.to_s =~ /1$/)
77
79
  raise ProcessingError
78
80
  else
@@ -80,6 +82,10 @@ module Yarn
80
82
  end
81
83
  end
82
84
 
85
+ def post_body
86
+ @request ? @request[:body].to_s : ""
87
+ end
88
+
83
89
  def return_response
84
90
  @session.puts "HTTP/1.1 #{@response.status} #{STATUS_CODES[@response.status]}"
85
91
  @session.puts @response.headers.map { |k,v| "#{k}: #{v}" }
@@ -101,9 +107,17 @@ module Yarn
101
107
  def read_request
102
108
  input = []
103
109
  while (line = @session.gets) do
104
- break if line.length <= 2
105
- input << line
110
+ length = line.gsub(/\D/,"") if line =~ /Content-Length/
111
+ if line == "\r\n"
112
+ input << line
113
+ input << @session.read(length.to_i) if length
114
+ break
115
+ else
116
+ input << line
117
+ end
106
118
  end
119
+ @session.close_read
120
+ debug "Done reading request"
107
121
  input.join
108
122
  end
109
123
 
@@ -141,12 +155,7 @@ module Yarn
141
155
 
142
156
  def read_file(path)
143
157
  file_contents = []
144
-
145
- File.open(path, "r") do |file|
146
- while (line = file.gets) do
147
- file_contents << line
148
- end
149
- end
158
+ File.open(path).each { |line| file_contents << line }
150
159
 
151
160
  file_contents
152
161
  end
@@ -5,55 +5,64 @@ module Yarn
5
5
 
6
6
  include Logging
7
7
 
8
- attr_accessor :host, :port, :socket, :socket_listener
8
+ attr_accessor :host, :port, :socket, :workers
9
9
 
10
- def initialize(app=nil,opts={})
10
+ def initialize(options={})
11
11
  # merge given options with default values
12
- options = {
12
+ opts = {
13
13
  output: $stdout,
14
14
  host: '127.0.0.1',
15
- port: 3000
16
- }.merge(opts)
15
+ port: 3000,
16
+ workers: 4,
17
+ rack: "off"
18
+ }.merge(options)
17
19
 
18
- @app = app
19
- @host,@port,$output = options[:host], options[:port], options[:output]
20
+ @app = nil
21
+ @app = load_rack_app(opts[:rack]) unless opts[:rack] == "off"
20
22
 
23
+ @host, @port, @num_workers = opts[:host], opts[:port], opts[:workers]
24
+ @workers = []
25
+ $output, $debug = opts[:output], opts[:debug]
26
+ end
27
+
28
+ def load_rack_app(app_path)
29
+ if File.exists?(app_path)
30
+ config_file = File.read(app_path)
31
+ rack_application = eval("Rack::Builder.new { #{config_file} }")
32
+ else
33
+ log "#{app_path} does not exist. Exiting."
34
+ Kernel::exit
35
+ end
36
+ end
37
+
38
+ def start
39
+ trap("INT") { stop }
21
40
  @socket = TCPServer.new(@host, @port)
41
+ log "Yarn started #{@num_workers} workers and is listening on #{@host}:#{@port}"
22
42
 
23
- @handler = @app ? RackHandler.new(@app, options) : RequestHandler.new(options)
43
+ init_workers
24
44
 
25
- log "Yarn started #{"w/ Rack " if opts[:rackup_file]}and accepting requests on #{@host}:#{@port}"
45
+ # Waits here for all processes to exit
46
+ Process.waitall
26
47
  end
27
48
 
28
- def start
29
- @socket_listener = Thread.new do
30
- loop do
31
- begin
49
+ def init_workers
50
+ @num_workers.times do
51
+ @workers << fork do
52
+ trap("INT") { exit }
53
+ loop do
54
+ handler ||= @app ? RackHandler.new(@app) : RequestHandler.new
32
55
  session = @socket.accept
33
- Thread.new { @handler.clone.run session }
34
- rescue Exception => e
35
- session.close
36
- log e.message
37
- log e.backtrace
56
+ handler.run session
38
57
  end
39
58
  end
40
59
  end
41
-
42
- begin
43
- @socket_listener.join
44
- rescue Interrupt => e
45
- log "Caught interrupt, stopping..."
46
- ensure
47
- stop
48
- end
49
60
  end
50
61
 
51
62
  def stop
52
- @socket.close if @socket
53
- @socket = nil
54
- @socket_listener.kill if @socket_listener
63
+ @socket.close if (@socket && !@socket.closed?)
55
64
 
56
- log "Server stopped"
65
+ log "Server stopped. Have a nice day!"
57
66
  end
58
67
  end
59
68
  end