RubyGems - iodine - Versions diffs - 0.1.21 → 0.2.0 - Mend

iodine 0.1.21 → 0.2.0

Potentially problematic release.

This version of iodine might be problematic. Click here for more details.

Files changed (105) hide show

checksums.yaml +4 -4
data/.gitignore +3 -2
data/.travis.yml +23 -2
data/CHANGELOG.md +9 -2
data/README.md +232 -179
data/Rakefile +13 -1
data/bin/config.ru +63 -0
data/bin/console +6 -0
data/bin/echo +42 -32
data/bin/http-hello +62 -0
data/bin/http-playground +124 -0
data/bin/playground +62 -0
data/bin/poc/Gemfile.lock +23 -0
data/bin/poc/README.md +37 -0
data/bin/poc/config.ru +66 -0
data/bin/poc/gemfile +1 -0
data/bin/poc/www/index.html +57 -0
data/bin/raw-rbhttp +35 -0
data/bin/raw_broadcast +66 -0
data/bin/test_with_faye +40 -0
data/bin/ws-broadcast +108 -0
data/bin/ws-echo +108 -0
data/exe/iodine +59 -0
data/ext/iodine/base64.c +264 -0
data/ext/iodine/base64.h +72 -0
data/ext/iodine/bscrypt-common.h +109 -0
data/ext/iodine/bscrypt.h +49 -0
data/ext/iodine/extconf.rb +41 -0
data/ext/iodine/hex.c +123 -0
data/ext/iodine/hex.h +70 -0
data/ext/iodine/http.c +200 -0
data/ext/iodine/http.h +128 -0
data/ext/iodine/http1.c +402 -0
data/ext/iodine/http1.h +56 -0
data/ext/iodine/http1_simple_parser.c +473 -0
data/ext/iodine/http1_simple_parser.h +59 -0
data/ext/iodine/http_request.h +128 -0
data/ext/iodine/http_response.c +1606 -0
data/ext/iodine/http_response.h +393 -0
data/ext/iodine/http_response_http1.h +374 -0
data/ext/iodine/iodine_core.c +641 -0
data/ext/iodine/iodine_core.h +70 -0
data/ext/iodine/iodine_http.c +615 -0
data/ext/iodine/iodine_http.h +19 -0
data/ext/iodine/iodine_websocket.c +430 -0
data/ext/iodine/iodine_websocket.h +21 -0
data/ext/iodine/libasync.c +552 -0
data/ext/iodine/libasync.h +117 -0
data/ext/iodine/libreact.c +347 -0
data/ext/iodine/libreact.h +244 -0
data/ext/iodine/libserver.c +912 -0
data/ext/iodine/libserver.h +435 -0
data/ext/iodine/libsock.c +950 -0
data/ext/iodine/libsock.h +478 -0
data/ext/iodine/misc.c +181 -0
data/ext/iodine/misc.h +76 -0
data/ext/iodine/random.c +193 -0
data/ext/iodine/random.h +48 -0
data/ext/iodine/rb-call.c +127 -0
data/ext/iodine/rb-call.h +60 -0
data/ext/iodine/rb-libasync.h +79 -0
data/ext/iodine/rb-rack-io.c +389 -0
data/ext/iodine/rb-rack-io.h +17 -0
data/ext/iodine/rb-registry.c +213 -0
data/ext/iodine/rb-registry.h +33 -0
data/ext/iodine/sha1.c +359 -0
data/ext/iodine/sha1.h +85 -0
data/ext/iodine/sha2.c +825 -0
data/ext/iodine/sha2.h +138 -0
data/ext/iodine/siphash.c +136 -0
data/ext/iodine/siphash.h +15 -0
data/ext/iodine/spnlock.h +235 -0
data/ext/iodine/websockets.c +696 -0
data/ext/iodine/websockets.h +120 -0
data/ext/iodine/xor-crypt.c +189 -0
data/ext/iodine/xor-crypt.h +107 -0
data/iodine.gemspec +25 -18
data/lib/iodine.rb +57 -58
data/lib/iodine/http.rb +0 -189
data/lib/iodine/protocol.rb +36 -245
data/lib/iodine/version.rb +1 -1
data/lib/rack/handler/iodine.rb +145 -2
metadata +115 -37
data/bin/core_http_test +0 -51
data/bin/em playground +0 -56
data/bin/hello_world +0 -75
data/bin/setup +0 -7
data/lib/iodine/client.rb +0 -5
data/lib/iodine/core.rb +0 -102
data/lib/iodine/core_init.rb +0 -143
data/lib/iodine/http/hpack.rb +0 -553
data/lib/iodine/http/http1.rb +0 -251
data/lib/iodine/http/http2.rb +0 -507
data/lib/iodine/http/rack_support.rb +0 -108
data/lib/iodine/http/request.rb +0 -462
data/lib/iodine/http/response.rb +0 -474
data/lib/iodine/http/session.rb +0 -143
data/lib/iodine/http/websocket_client.rb +0 -335
data/lib/iodine/http/websocket_handler.rb +0 -101
data/lib/iodine/http/websockets.rb +0 -336
data/lib/iodine/io.rb +0 -56
data/lib/iodine/logging.rb +0 -46
data/lib/iodine/settings.rb +0 -158
data/lib/iodine/ssl_connector.rb +0 -48
data/lib/iodine/timers.rb +0 -95

data/bin/poc/README.md ADDED Viewed

@@ -0,0 +1,37 @@
+# A proof of concept for Rack's `env['rack.websocket']`
+This is a proof of concept for Rack based Websocket connections, showing how the Rack API can be adjusted to support server native real-time connections.
+The chosen proof of concept was the ugliest chatroom I could find.
+Although my hope is that Rack will adopt the concept and make `env['rack.websocket?']` and `env['rack.websocket']` part of it's standard, at the moment it's an Iodine specific feature, implemented using `env['iodine.websocket']`.
+## Install
+Install required gems using:
+```sh
+bundler install
+```
+## Run
+Run this application single threaded:
+```sh
+bundler exec iodine -- -www ./www
+```
+Or both multi-threaded and forked (you'll notice that memory barriers for forked processes prevent websocket broadcasting from reaching websockets connected to a different process).
+```sh
+bundler exec iodine -- -www ./www -t 16 -w 4
+```
+## Further reading
+* https://github.com/rack/rack/issues/1093
+* https://bowild.wordpress.com/2016/07/31/the-dark-side-of-the-rack
+* https://github.com/boazsegev/iodine

data/bin/poc/config.ru ADDED Viewed

@@ -0,0 +1,66 @@
+# The Rack Application container
+module MyRackApplication
+  # Rack applications use the `call` callback to handle HTTP requests.
+  def self.call(env)
+    # if upgrading...
+    if env['HTTP_UPGRADE'.freeze] =~ /websocket/i
+      # We can assign a class or an instance that implements callbacks.
+      # We will assign an object, passing it the request information (`env`)
+      env['iodine.websocket'.freeze] = MyWebsocket.new(env)
+      # Rack responses must be a 3 item array
+      # [status, {http: :headers}, ["response body"]]
+      return [0, {}, []]
+    end
+    # a semi-regualr HTTP response
+    out = File.open File.expand_path('../www/index.html', __FILE__)
+    [200, { 'X-Sendfile' => File.expand_path('../www/index.html', __FILE__),
+            'Content-Length' => out.size }, out]
+  end
+end
+# The Websocket Callback Object
+class MyWebsocket
+  # this is optional, but I wanted the object to have the nickname provided in
+  # the HTTP request
+  def initialize(env)
+    # we need to change the ASCI Rack encoding to UTF-8,
+    # otherwise everything with the nickname will be a binary "blob" in the
+    # Javascript layer
+    @nickname = env['PATH_INFO'][1..-1].force_encoding 'UTF-8'
+  end
+  # A classic websocket callback, called when the connection is opened and
+  # linked to this object
+  def on_open
+    puts 'We have a websocket connection'
+  end
+  # A classic websocket callback, called when the connection is closed
+  # (after disconnection).
+  def on_close
+    puts "Bye Bye... #{count} connections left..."
+  end
+  # A server-side niceness, called when the server if shutting down,
+  # to gracefully disconnect (before disconnection).
+  def on_shutdown
+    write 'The server is shutting down, goodbye.'
+  end
+  def on_message(data)
+    puts "got message: #{data} encoded as #{data.encoding}"
+    # data is a temporary string, it's buffer cleared as soon as we return.
+    # So we make a copy with the desired format.
+    tmp = "#{@nickname}: #{data}"
+    # The `write` method was added by the server and writes to the current
+    # connection
+    write tmp
+    puts "broadcasting #{tmp.bytesize} bytes with encoding #{tmp.encoding}"
+    # `each` was added by the server and excludes this connection
+    # (each except self).
+    each { |h| h.write tmp }
+  end
+end
+# `run` is a Rack API command, telling Rack where the `call(env)` callback is located.
+run MyRackApplication

data/bin/poc/gemfile ADDED Viewed

	@@ -0,0 +1 @@
1	+ gem 'iodine', git: 'https://github.com/boazsegev/iodine.git'

data/bin/poc/www/index.html ADDED Viewed

@@ -0,0 +1,57 @@
+<!DOCTYPE html>
+<html>
+<head>
+    <script src="https://ajax.googleapis.com/ajax/libs/jquery/2.1.4/jquery.min.js"></script>
+    <script>
+        ws = NaN
+        handle = ''
+        function onsubmit(e) {
+            e.preventDefault();
+            if($('#text')[0].value == '') {return false}
+            if(ws && ws.readyState == 1) {
+                ws.send($('#text')[0].value);
+                $('#text')[0].value = '';
+            } else {
+                handle = $('#text')[0].value
+                var url = (window.location.protocol.match(/https/) ? 'wss' : 'ws') +
+                            '://' + window.document.location.host +
+                            '/' + $('#text')[0].value
+                ws = new WebSocket(url)
+                ws.onopen = function(e) {
+                    output("<b>Connected :-)</b>");
+                    $('#text')[0].value = '';
+                    $('#text')[0].placeholder = 'your message';
+                }
+                ws.onclose = function(e) {
+                    output("<b>Disonnected :-/</b>")
+                    $('#text')[0].value = '';
+                    $('#text')[0].placeholder = 'nickname';
+                    $('#text')[0].value = handle
+                }
+                ws.onmessage = function(e) {
+                    output(e.data);
+                }
+            }
+            return false;
+        }
+        function output(data) {
+            $('#output').append("<li>" + data + "</li>")
+            $('#output').animate({ scrollTop:
+                        $('#output')[0].scrollHeight }, "slow");
+        }
+    </script>
+    <style>
+    html, body {width:100%; height: 100%; background-color: #ddd; color: #111;}
+    h3, form {text-align: center;}
+    input {background-color: #fff; color: #111; padding: 0.3em;}
+    </style>
+</head><body>
+  <h3>The Ugly Chatroom POC</h3>
+    <form id='form'>
+        <input type='text' id='text' name='text' placeholder='nickname'></input>
+        <input type='submit' value='send'></input>
+    </form>
+    <script> $('#form')[0].onsubmit = onsubmit </script>
+    <ul id='output'></ul>
+</body>
+</html>

data/bin/raw-rbhttp ADDED Viewed

@@ -0,0 +1,35 @@
+#!/usr/bin/env ruby
+# this will compile Iodine and run a raw sockets server that emulates HTTP (without parsing any incoming requests).
+# # test using:
+# ab -n 100000 -c 200 -k http://127.0.0.1:3000/
+Dir.chdir(File.expand_path(File.join('..', '..'), __FILE__))
+puts `rake clean`
+puts `rake compile`
+require 'benchmark'
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', 'lib'), __FILE__)
+require 'bundler/setup'
+require 'iodine'
+class HttpProtocol
+  @timeout = 5
+  # `on_message` is an optional alternative to the `on_data` callback.
+  # `on_message` has a 1Kb buffer that recycles itself for memory optimization.
+  def on_data
+    # writing will never block and will use a buffer written in C when needed.
+    data = nil
+    while (data = read)
+      write "HTTP/1.1 200 OK\r\nConnection: keep-alive\r\nKeep-Alive: timeout=1\r\nContent-Length: 12\r\n\r\nHello World!"
+    end
+  end
+end
+puts "thread #{Iodine.threads}"
+# Listen
+Iodine.listen '3000', HttpProtocol
+Iodine.threads = 7
+puts "now, thread #{Iodine.threads}"
+Iodine.start

data/bin/raw_broadcast ADDED Viewed

@@ -0,0 +1,66 @@
+#!/usr/bin/env ruby
+# this will compile Iodine and run a raw sockets broadcasting (chatroom) server.
+# # test using:
+# telnet localhost 3000
+Dir.chdir(File.expand_path(File.join('..', '..'), __FILE__))
+puts `rake clean`
+puts `rake compile`
+require 'benchmark'
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', 'lib'), __FILE__)
+require 'bundler/setup'
+require 'iodine'
+# You can add fixtures and/or initialization code here to make experimenting
+# with your gem easier. You can also use a different console, if you like.
+# (If you use this, don't forget to add pry to your Gemfile!)
+# require "pry"
+# Pry.start
+# test the upgrade protocol using the ShoooProtocol
+class ShoooProtocol
+  @timeout = 10
+  def ping
+    write "Wha, ya' still here?! Go away!\n"
+    close
+  end
+  def on_message(buffer)
+    write "what do you mean - #{buffer.strip} ?!\nShoo, away with you!\n"
+    close
+  end
+end
+# the broadcasting protocol
+class EchoProtocol
+  def self.on_start
+    Iodine.run_every(1000) { puts "#{Iodine.count} clients connected." }
+    puts "We have 1 timer, and #{Iodine.count} connections."
+  end
+  # `on_message` is an optional alternative to the `on_data` callback.
+  # `on_message` has a 1Kb buffer that recycles itself for memory optimization.
+  def on_message(buffer)
+    # writing will never block and will use a buffer written in C when needed.
+    write buffer
+    puts buffer.dump
+    # close will be performed only once all the data in the write buffer
+    # was sent. use `force_close` to close early.
+    close if buffer =~ /^bye[\r\n]/i
+    upgrade ShoooProtocol if buffer =~ /^shoo[\r\n]/i
+    # use buffer.dup to save the data from being recycled once we return.
+    data = "Someone said: #{buffer}"
+    # run asynchronous tasks with ease
+    each { |c| puts c; c.write data }
+  end
+end
+# create the server object and setup any settings we might need.
+Iodine.listen 3000, EchoProtocol
+Iodine.threads = 10
+Iodine.processes = 1
+Iodine.start

data/bin/test_with_faye ADDED Viewed

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+Dir.chdir(File.expand_path(File.join('..', '..'), __FILE__))
+puts `rake clean`
+puts `rake compile`
+require 'benchmark'
+require 'faye/websocket'
+# require 'thin'
+# Faye::WebSocket.load_adapter('thin')
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', 'lib'), __FILE__)
+require 'bundler/setup'
+require 'iodine'
+Iodine::Rack.app = lambda do |env|
+  if Faye::WebSocket.websocket?(env)
+    ws = Faye::WebSocket.new(env)
+    ws.on :message do |event|
+      ws.send(event.data.to_s)
+    end
+    ws.on :close do |event|
+      p [:close, event.code, event.reason]
+      ws = nil
+    end
+    # Return async Rack response
+    return ws.rack_response
+  else
+    # Normal HTTP request
+    [200, { 'Content-Type' => 'text/plain' }, ["Hello\n"]]
+  end
+end
+Iodine.start
+# ws = new WebSocket("ws://localhost:3000"); ws.onmessage = function(e) {console.log("Got message!"); console.log(e.data);}; ws.onclose = function(e) {console.log("closed")}; ws.onopen = function(e) {ws.send("hi");};

data/bin/ws-broadcast ADDED Viewed

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+Dir.chdir(File.expand_path(File.join('..', '..'), __FILE__))
+puts `rake clean`
+puts `rake compile`
+require 'benchmark'
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', 'lib'), __FILE__)
+require 'bundler/setup'
+require 'iodine'
+require 'rack'
+class WSEcho
+  def self.call(env)
+    if env['HTTP_UPGRADE'.freeze] =~ /websocket/i
+      env['iodine.websocket'.freeze] = WSEcho
+      return [0, {}, []]
+    end
+    out = "ENV:\r\n#{env.to_a.map { |h| "#{h[0]}: #{h[1]}" } .join "\n"}"
+    request = Rack::Request.new(env)
+    pr_raise if request.path_info =~ /raise[.]*/
+    out += "\nRequest Path: #{request.path_info}\nParams:\r\n#{request.params.to_a.map { |h| "#{h[0]}: #{h[1]}" } .join "\n"}" unless request.params.empty?
+    [200, { 'Content-Length' => out.length }, [out]]
+  end
+  def on_open
+    puts 'We have a websocket connection'
+  end
+  def on_close
+    puts "Bye Bye... only #{count} left..."
+  end
+  def on_shutdown
+    puts "I'm shutting down #{self}"
+  end
+  def on_message(data)
+    puts "got message: #{data}"
+    write data
+    data_cp = "Broadcast: #{data}"
+    # puts "broadcasting #{data_cp.bytesize} bytes"
+    each { |h| h.write data_cp }
+  end
+  def self.pr_raise
+    raise 'hell'
+  end
+end
+# create the server object and setup any settings we might need.
+Iodine.threads ||= 4
+Iodine.processes ||= 1
+Iodine::Rack.public = nil
+Iodine::Rack.log = 1
+Iodine::Rack.app = WSEcho
+# puts "Press enter to start (#{Process.pid})"
+# gets
+Iodine.start
+# server.on_start do
+#   # server.run {puts "I'm running!"}
+#   # server.run_after(5000) {puts "5 seconds have passed."}
+#   server.run_every(1000) { puts "#{server.count} clients connected." }
+#   # server.run_every(10000) do
+#   #   begin
+#   #     puts "making a system call"
+#   #     puts `ab -n 100000 -c 200 -k http://127.0.0.1:3000/`
+#   #   rescue => e
+#   #     l = Logger.new STDOUT
+#   #     l.error e
+#   #   end
+#   # end
+# end
+#
+#   Iodine.run_every(1000) {puts "#{server.connection_count} clients connected."}
+# def nag
+#   puts `ab -n 200000 -c 2000 -k http://127.0.0.1:3000/`
+#   sleep 2
+# end
+#
+# nag while true
+#
+# def nag
+#   puts `wrk -c2000 -d10 -t4 http://localhost:3000/`
+#   sleep 3
+# end
+#
+# nag while true
+# ab -n 100000 -c 200 -k http://127.0.0.1:3000/
+# ab -n 100000 -c 4000 -k http://127.0.0.1:3000/
+# ab -n 1000000 -c 20000 -k http://127.0.0.1:3000/
+# ~/ruby/wrk/wrk -c400 -d10 -t12 http://localhost:3000/
+# wrk -c200 -d4 -t12 http://localhost:3000/
+# RACK_ENV="production" rackup -p 3000 -s iodine
+# thor --amount 5000 ws://localhost:3000/echo
+# thor --amount 5000 ws://localhost:3000/broadcast
+# ws = new WebSocket("ws://localhost:3000"); ws.onmessage = function(e) {console.log("Got message!"); console.log(e.data);}; ws.onclose = function(e) {console.log("closed")}; ws.onopen = function(e) {ws.send("hi");};
+# for(i = 0; i< 256; i++) {
+#   ws = new WebSocket("ws://localhost:3000/");
+#   ws.onmessage = function(e) {console.log("Got message!"); console.log(e.data); window.setTimeout(function(iws) {iws.send("leaving soon...");}, 3000, e.target); window.setTimeout(function(iws) {iws.close();}, 5000, e.target); };
+#   ws.onclose = function(e) {console.log("closed")};
+#   ws.onopen = function(e) {e.target.send("hi");};
+# };

data/bin/ws-echo ADDED Viewed

@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+Dir.chdir(File.expand_path(File.join('..', '..'), __FILE__))
+puts `rake clean`
+puts `rake compile`
+require 'benchmark'
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', 'lib'), __FILE__)
+require 'bundler/setup'
+require 'iodine'
+require 'rack'
+class WSEcho
+  def self.call(env)
+    if env['upgrade.websocket?'.freeze] # && env['HTTP_UPGRADE'.freeze] =~ /websocket/i
+      env['upgrade.websocket'.freeze] = WSEcho
+      return [0, {}, []]
+    end
+    out = "ENV:\r\n#{env.to_a.map { |h| "#{h[0]}: #{h[1]}" } .join "\n"}\n"
+    request = Rack::Request.new(env)
+    out += "\nRequest Path: #{request.path_info}\nParams:\r\n#{request.params.to_a.map { |h| "#{h[0]}: #{h[1]}" } .join "\n"}\n" unless request.params.empty?
+    [200, { 'Content-Length' => out.length, 'Content-Type' => 'text/plain; charset=UTF-8;' }, [out]]
+  end
+  def on_open
+    puts 'We have a websocket connection'
+    Iodine::Websocket.defer(uuid) { |ws| puts "This websocket's uuid is #{ws.uuid}" }
+  end
+  def on_close
+    puts "Bye Bye... only #{count} left..."
+  end
+  def on_shutdown
+    puts "I'm shutting down #{self}"
+  end
+  def on_message(data)
+    puts "got message: #{data}"
+    write data
+  end
+  def echo(data)
+    write "echo: #{data}"
+  end
+end
+# create the server object and setup any settings we might need.
+Iodine.threads ||= 4
+Iodine.processes ||= 1
+Iodine::Rack.public = nil
+Iodine::Rack.log = 1
+count = 2
+Iodine::Rack.app = WSEcho
+Iodine.start
+# server.on_http= Proc.new do |env|
+#   # [200, {"Content-Length".freeze => "12".freeze}, ["Hello World!".freeze]];
+#   if env["HTTP_UPGRADE".freeze] =~ /websocket/i.freeze
+#     env['iodine.websocket'.freeze] = WSEcho.new
+#     [0,{}, []]
+#   else
+#     req = Rack::Request.new env
+#     res = Rack::Response.new
+#     res.write "Hello World!".freeze
+#     res.to_a
+#   end
+# end
+# server.on_start do
+#   server.run_every(1000) {puts "#{server.connection_count} clients connected."}
+# end
+# puts "Press enter to start (#{Process.pid})"
+# gets
+# def nag
+#   puts `ab -n 200000 -c 2000 -k http://127.0.0.1:3000/`
+#   sleep 2
+# end
+#
+# nag while true
+#
+# def nag
+#   puts `wrk -c2000 -d10 -t4 http://localhost:3000/`
+#   sleep 3
+# end
+#
+# nag while true
+# ab -n 100000 -c 200 -k http://127.0.0.1:3000/
+# ab -n 100000 -c 4000 -k http://127.0.0.1:3000/
+# ab -n 1000000 -c 20000 -k http://127.0.0.1:3000/
+# ~/ruby/wrk/wrk -c400 -d10 -t12 http://localhost:3000/
+# wrk -c200 -d4 -t12 http://localhost:3000/
+# ab -n 2000 -c 20 -H "Connection: close" http://127.0.0.1:3000/
+# RACK_ENV="production" rackup -p 3000 -s iodine
+# thor --amount 5000 ws://localhost:3000/echo
+# thor --amount 5000 ws://localhost:3000/broadcast
+# ws = new WebSocket("ws://localhost:3000"); ws.onmessage = function(e) {console.log("Got message!"); console.log(e.data);}; ws.onclose = function(e) {console.log("closed")}; ws.onopen = function(e) {ws.send("hi");};
+# for(i = 0; i< 256; i++) {
+#   ws = new WebSocket("ws://localhost:3000");
+#   ws.onmessage = function(e) {console.log("Got message!"); console.log(e.data); e.target.close(); };
+#   ws.onclose = function(e) {console.log("closed")};
+#   ws.onopen = function(e) {e.target.send("hi");};
+# };