RubyGems - tipi - Versions diffs - 0.31 → 0.36 - Mend

tipi 0.31 → 0.36

Files changed (50) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +29 -0
data/Gemfile.lock +10 -4
data/LICENSE +1 -1
data/TODO.md +13 -47
data/bin/tipi +13 -0
data/df/agent.rb +63 -0
data/df/etc_benchmark.rb +15 -0
data/df/multi_agent_supervisor.rb +87 -0
data/df/multi_client.rb +84 -0
data/df/routing_benchmark.rb +60 -0
data/df/sample_agent.rb +89 -0
data/df/server.rb +54 -0
data/df/sse_page.html +29 -0
data/df/stress.rb +24 -0
data/df/ws_page.html +38 -0
data/examples/http_request_ws_server.rb +34 -0
data/examples/http_server.rb +6 -6
data/examples/http_server_forked.rb +4 -5
data/examples/http_server_form.rb +23 -0
data/examples/http_server_throttled_accept.rb +23 -0
data/examples/http_unix_socket_server.rb +17 -0
data/examples/http_ws_server.rb +10 -12
data/examples/routing_server.rb +34 -0
data/examples/websocket_client.rb +1 -2
data/examples/websocket_demo.rb +4 -2
data/examples/ws_page.html +1 -2
data/lib/tipi.rb +7 -5
data/lib/tipi/config_dsl.rb +153 -0
data/lib/tipi/configuration.rb +1 -1
data/lib/tipi/digital_fabric.rb +7 -0
data/lib/tipi/digital_fabric/agent.rb +225 -0
data/lib/tipi/digital_fabric/agent_proxy.rb +265 -0
data/lib/tipi/digital_fabric/executive.rb +100 -0
data/lib/tipi/digital_fabric/executive/index.html +69 -0
data/lib/tipi/digital_fabric/protocol.rb +90 -0
data/lib/tipi/digital_fabric/request_adapter.rb +48 -0
data/lib/tipi/digital_fabric/service.rb +230 -0
data/lib/tipi/http1_adapter.rb +59 -37
data/lib/tipi/http2_adapter.rb +5 -3
data/lib/tipi/http2_stream.rb +19 -7
data/lib/tipi/rack_adapter.rb +11 -3
data/lib/tipi/version.rb +1 -1
data/lib/tipi/websocket.rb +33 -13
data/test/helper.rb +1 -2
data/test/test_http_server.rb +3 -2
data/test/test_request.rb +108 -0
data/tipi.gemspec +7 -3
metadata +59 -7
data/lib/tipi/request.rb +0 -118

data/examples/http_ws_server.rb CHANGED Viewed

@@ -5,14 +5,14 @@ require 'tipi'
 require 'tipi/websocket'
 def ws_handler(conn)
-  timer = spin do
-    throttled_loop(1) do
-      conn << Time.now.to_s
-    end
+  timer = spin_loop(interval: 1) do
+    conn << Time.now.to_s
   end
   while (msg = conn.recv)
     conn << "you said: #{msg}"
   end
+rescue Exception => e
+  p e
 ensure
   timer.stop
 end
@@ -21,17 +21,15 @@ opts = {
   reuse_addr:  true,
   dont_linger: true,
   upgrade:     {
-    websocket: Polyphony::Websocket.handler(&method(:ws_handler))
+    websocket: Tipi::Websocket.handler(&method(:ws_handler))
   }
 }
 HTML = IO.read(File.join(__dir__, 'ws_page.html'))
-spin do
-  Tipi.serve('0.0.0.0', 1234, opts) do |req|
-    req.respond(HTML, 'Content-Type' => 'text/html')
-  end
-end
 puts "pid: #{Process.pid}"
-puts 'Listening on port 1234...'
+puts 'Listening on port 4411...'
+Tipi.serve('0.0.0.0', 4411, opts) do |req|
+  req.respond(HTML, 'Content-Type' => 'text/html')
+end

data/examples/routing_server.rb ADDED Viewed

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+require 'bundler/setup'
+require 'tipi'
+opts = {
+  reuse_addr:  true,
+  dont_linger: true
+}
+puts "pid: #{Process.pid}"
+puts 'Listening on port 4411...'
+app = Tipi.route do |r|
+  r.root do
+    r.redirect '/hello'
+  end
+  r.on 'hello' do
+    r.get 'world' do
+      r.respond 'Hello world'
+    end
+    r.get do
+      r.respond 'Hello'
+    end
+    r.post do
+      puts 'Someone said Hello'
+      r.redirect '/'
+    end
+  end
+end
+spin do
+  Tipi.serve('0.0.0.0', 4411, opts, &app)
+end.await

data/examples/websocket_client.rb CHANGED Viewed

@@ -26,8 +26,7 @@ class WebsocketClient
   end
   def receive
-    loop do
-      data = @socket.readpartial(8192)
+    @socket.read_loop do |data|
       @reader << data
       parsed = @reader.next
       return parsed if parsed

data/examples/websocket_demo.rb CHANGED Viewed

@@ -30,8 +30,10 @@ class WebsocketClient
   end
   def receive
-    loop do
-      data = @socket.readpartial(8192)
+    parsed = @reader.next
+    return parsed if parsed
+    @socket.read_loop do |data|
       @reader << data
       parsed = @reader.next
       return parsed if parsed

data/examples/ws_page.html CHANGED Viewed

@@ -6,14 +6,13 @@
 <body>
   <script>
     var connect = function () {
-      var exampleSocket = new WebSocket("ws://localhost:1234");
+      var exampleSocket = new WebSocket("wss://dev.realiteq.net/");
       exampleSocket.onopen = function (event) {
         document.querySelector('#status').innerText = 'connected';
         exampleSocket.send("Can you hear me?");
       };
       exampleSocket.onclose = function (event) {
-        console.log('onclose');
         document.querySelector('#status').innerText = 'disconnected';
         setTimeout(function () {
           // exampleSocket.removeAllListeners();

data/lib/tipi.rb CHANGED Viewed

@@ -28,10 +28,8 @@ module Tipi
     end
     def accept_loop(server, opts, &handler)
-      loop do
-        client = server.accept
+      server.accept_loop do |client|
         spin { client_loop(client, opts, &handler) }
-        snooze
       rescue OpenSSL::SSL::SSLError
         # disregard
       end
@@ -42,14 +40,18 @@ module Tipi
       adapter = protocol_adapter(client, opts)
       adapter.each(&handler)
     ensure
-      client.close
+      client.close rescue nil
     end
     def protocol_adapter(socket, opts)
       use_http2 = socket.respond_to?(:alpn_protocol) &&
-      socket.alpn_protocol == H2_PROTOCOL
+                  socket.alpn_protocol == H2_PROTOCOL
       klass = use_http2 ? HTTP2Adapter : HTTP1Adapter
       klass.new(socket, opts)
     end
+    def route(&block)
+      proc { |req| req.route(&block) }
+    end
   end
 end

data/lib/tipi/config_dsl.rb ADDED Viewed

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+module Tipi
+  module Configuration
+    class Interpreter
+      # make_blank_slate
+      def initialize(assembler)
+        @assembler = assembler
+      end
+      def gzip_response
+        @assembler.emit 'req = Tipi::GZip.wrap(req)'
+      end
+      def log(out)
+        @assembler.wrap_current_frame 'logger.log_request(req) do |req|'
+      end
+      def error(&block)
+        assembler.emit_exception_handler &block
+      end
+      def match(pattern, &block)
+        @assembler.emit_conditional "if req.path =~ #{pattern.inspect}", &block
+      end
+    end
+    class Assembler
+      def self.from_source(code)
+        new.from_source code
+      end
+      def from_source(code)
+        @stack = [new_frame]
+        @app_procs = {}
+        @interpreter = Interpreter.new self
+        @interpreter.instance_eval code
+        loop do
+          frame = @stack.pop
+          return assemble_app_proc(frame).join("\n") if @stack.empty?
+          @stack.last[:body] << assemble_frame(frame)
+        end
+      end
+      def new_frame
+        {
+          prelude: [],
+          body: []
+        }
+      end
+      def add_frame(&block)
+        @stack.push new_frame
+        yield
+      ensure
+        frame = @stack.pop
+        emit assemble(frame)
+      end
+      def wrap_current_frame(head)
+        frame = @stack.pop
+        wrapper = new_frame
+        wrapper[:body] << head
+        @stack.push wrapper
+        @stack.push frame
+      end
+      def emit(code)
+        @stack.last[:body] << code
+      end
+      def emit_prelude(code)
+        @stack.last[:prelude] << code
+      end
+      def emit_exception_handler(&block)
+        proc_id = add_app_proc block
+        @stack.last[:rescue_proc_id] = proc_id
+      end
+      def emit_block(conditional, &block)
+        proc_id = add_app_proc block
+        @stack.last[:branched] = true
+        emit conditional
+        add_frame &block
+      end
+      def add_app_proc(proc)
+        id = :"proc#{@app_procs.size}"
+        @app_procs[id] = proc
+        id
+      end
+      def assemble_frame(frame)
+        indent = 0
+        lines = []
+        emit_code lines, frame[:prelude], indent
+        if frame[:rescue_proc_id]
+          emit_code lines, 'begin', indent
+          indent += 1
+        end
+        emit_code lines, frame[:body], indent
+        if frame[:rescue_proc_id]
+          emit_code lines, 'rescue => e', indent
+          emit_code lines, "  app_procs[#{frame[:rescue_proc_id].inspect}].call(req, e)", indent
+          emit_code lines, 'end', indent
+          indent -= 1
+        end
+        lines
+      end
+      def assemble_app_proc(frame)
+        indent = 0
+        lines = []
+        emit_code lines, frame[:prelude], indent
+        emit_code lines, 'proc do |req|', indent
+        emit_code lines, frame[:body], indent + 1
+        emit_code lines, 'end', indent
+        lines
+      end
+      def emit_code(lines, code, indent)
+        if code.is_a? Array
+          code.each { |l| emit_code lines, l, indent + 1 }
+        else
+          lines << (indent_line code, indent)
+        end
+      end
+      @@indents = Hash.new { |h, k| h[k] =  '  ' * k }
+      def indent_line(code, indent)
+        indent == 0 ? code : "#{ @@indents[indent] }#{code}"
+      end
+    end
+  end
+end
+def assemble(code)
+  Tipi::Configuration::Assembler.from_source(code)
+end
+code = assemble <<~RUBY
+gzip_response
+log STDOUT
+RUBY
+puts code

data/lib/tipi/configuration.rb CHANGED Viewed

@@ -19,7 +19,7 @@ module Tipi
       end
       def start_listeners(config)
-        puts "listening on port 1234"
+        puts "Listening on port 1234"
         @server = Polyphony::Net.tcp_listen('0.0.0.0', 1234, { reuse_addr: true, dont_linger: true })
       end

data/lib/tipi/digital_fabric.rb ADDED Viewed

@@ -0,0 +1,7 @@
+module DigitalFabric
+end
+::DF = DigitalFabric
+require_relative 'digital_fabric/service'
+require_relative 'digital_fabric/agent_proxy'

data/lib/tipi/digital_fabric/agent.rb ADDED Viewed

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+require_relative './protocol'
+require_relative './request_adapter'
+require 'msgpack'
+require 'tipi/websocket'
+require 'tipi/request'
+module DigitalFabric
+  class Agent
+    def initialize(server_url, route, token)
+      @server_url = server_url
+      @route = route
+      @token = token
+      @requests = {}
+      @long_running_requests = {}
+      @name = '<unknown>'
+    end
+    class TimeoutError < RuntimeError
+    end
+    class GracefulShutdown < RuntimeError
+    end
+    def run
+      @fiber = Fiber.current
+      @keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      while true
+        connect_and_process_incoming_requests
+        return if @shutdown
+        sleep 5
+      end
+    ensure
+      @keep_alive_timer.stop
+    end
+    def connect_and_process_incoming_requests
+      # log 'Connecting...'
+      @socket = connect_to_server
+      @last_recv = @last_send = Time.now
+      df_upgrade
+      @connected = true
+      @msgpack_reader = MessagePack::Unpacker.new
+      process_incoming_requests
+    rescue IOError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, TimeoutError
+      log 'Disconnected' if @connected
+      @connected = nil
+    end
+    def connect_to_server
+      if @server_url =~ /^([^\:]+)\:(\d+)$/
+        host = Regexp.last_match(1)
+        port = Regexp.last_match(2)
+        Polyphony::Net.tcp_connect(host, port)
+      else
+        UNIXSocket.new(@server_url)
+      end
+    end
+    UPGRADE_REQUEST = <<~HTTP
+    GET / HTTP/1.1
+    Host: localhost
+    Connection: upgrade
+    Upgrade: df
+    DF-Token: %s
+    DF-Mount: %s
+    HTTP
+    def df_upgrade
+      @socket << format(UPGRADE_REQUEST, @token, mount_point)
+      while (line = @socket.gets)
+        break if line.chomp.empty?
+      end
+      # log 'Connection upgraded'
+    end
+    def mount_point
+      if @route[:host]
+        "host=#{@route[:host]}"
+      elsif @route[:path]
+        "path=#{@route[:path]}"
+      else
+        nil
+      end
+    end
+    def log(msg)
+      puts "#{Time.now} (#{@name}) #{msg}"
+    end
+    def process_incoming_requests
+      @socket.feed_loop(@msgpack_reader, :feed_each) do |msg|
+        recv_df_message(msg)
+        return if @shutdown && @requests.empty?
+      end
+    rescue IOError, SystemCallError, TimeoutError
+      # ignore
+    end
+    def keep_alive
+      return unless @connected
+      now = Time.now
+      if now - @last_send >= Protocol::SEND_TIMEOUT
+        send_df_message(Protocol.ping)
+      end
+      # if now - @last_recv >= Protocol::RECV_TIMEOUT
+      #   raise TimeoutError
+      # end
+    rescue IOError, SystemCallError => e
+      # transmit exception to fiber running the agent
+      @fiber.raise(e)
+    end
+    def recv_df_message(msg)
+      @last_recv = Time.now
+      case msg['kind']
+      when Protocol::SHUTDOWN
+        recv_shutdown
+      when Protocol::HTTP_REQUEST
+        recv_http_request(msg)
+      when Protocol::HTTP_REQUEST_BODY
+        recv_http_request_body(msg)
+      when Protocol::WS_REQUEST
+        recv_ws_request(msg)
+      when Protocol::CONN_DATA, Protocol::CONN_CLOSE,
+           Protocol::WS_DATA, Protocol::WS_CLOSE
+        fiber = @requests[msg['id']]
+        fiber << msg if fiber
+      end
+    end
+    def send_df_message(msg)
+      # we mark long-running requests by applying simple heuristics to sent DF
+      # messages. This is so we can correctly stop long-running requests
+      # upon graceful shutdown
+      if is_long_running_request_response?(msg)
+        id = msg[:id]
+        @long_running_requests[id] = @requests[id]
+      end
+      @last_send = Time.now
+      @socket << msg.to_msgpack
+    end
+    def is_long_running_request_response?(msg)
+      case msg[:kind]
+      when Protocol::HTTP_UPGRADE
+        true
+      when Protocol::HTTP_RESPONSE
+        msg[:body] && !msg[:complete]
+      end
+    end
+    def recv_shutdown
+      # puts "Received shutdown message (#{@requests.size} pending requests)"
+      # puts "  (Long running requests: #{@long_running_requests.size})"
+      @shutdown = true
+      @long_running_requests.values.each { |f| f.terminate(true) }
+    end
+    def recv_http_request(msg)
+      req = prepare_http_request(msg)
+      id = msg['id']
+      @requests[id] = spin do
+        http_request(req)
+      rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
+        # ignore
+      rescue Polyphony::Terminate
+        req.respond(nil, { ':status' => Qeweney::Status::SERVICE_UNAVAILABLE }) if Fiber.current.graceful_shutdown?
+      ensure
+        @requests.delete(id)
+        @long_running_requests.delete(id)
+        @fiber.terminate if @shutdown && @requests.empty?
+      end
+    end
+    def prepare_http_request(msg)
+      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
+      req.buffer_body_chunk(msg['body']) if msg['body']
+      req.complete! if msg['complete']
+      req
+    end
+    def recv_http_request_body(msg)
+      fiber = @requests[msg['id']]
+      return unless fiber
+      fiber << msg['body']
+    end
+    def get_http_request_body(id, limit)
+      send_df_message(Protocol.http_get_request_body(id, limit))
+      receive
+    end
+    def recv_ws_request(msg)
+      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
+      id = msg['id']
+      @requests[id] = @long_running_requests[id] = spin do
+        ws_request(req)
+      rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
+        # ignore
+      ensure
+        @requests.delete(id)
+        @long_running_requests.delete(id)
+        @fiber.terminate if @shutdown && @requests.empty?
+      end
+    end
+    # default handler for HTTP request
+    def http_request(req)
+      req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
+    end
+    # default handler for WS request
+    def ws_request(req)
+      req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
+    end
+  end
+end