RubyGems - tipi - Versions diffs - 0.31 → 0.36 - Mend

tipi 0.31 → 0.36

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (50) hide show

checksums.yaml +4 -4
data/CHANGELOG.md +29 -0
data/Gemfile.lock +10 -4
data/LICENSE +1 -1
data/TODO.md +13 -47
data/bin/tipi +13 -0
data/df/agent.rb +63 -0
data/df/etc_benchmark.rb +15 -0
data/df/multi_agent_supervisor.rb +87 -0
data/df/multi_client.rb +84 -0
data/df/routing_benchmark.rb +60 -0
data/df/sample_agent.rb +89 -0
data/df/server.rb +54 -0
data/df/sse_page.html +29 -0
data/df/stress.rb +24 -0
data/df/ws_page.html +38 -0
data/examples/http_request_ws_server.rb +34 -0
data/examples/http_server.rb +6 -6
data/examples/http_server_forked.rb +4 -5
data/examples/http_server_form.rb +23 -0
data/examples/http_server_throttled_accept.rb +23 -0
data/examples/http_unix_socket_server.rb +17 -0
data/examples/http_ws_server.rb +10 -12
data/examples/routing_server.rb +34 -0
data/examples/websocket_client.rb +1 -2
data/examples/websocket_demo.rb +4 -2
data/examples/ws_page.html +1 -2
data/lib/tipi.rb +7 -5
data/lib/tipi/config_dsl.rb +153 -0
data/lib/tipi/configuration.rb +1 -1
data/lib/tipi/digital_fabric.rb +7 -0
data/lib/tipi/digital_fabric/agent.rb +225 -0
data/lib/tipi/digital_fabric/agent_proxy.rb +265 -0
data/lib/tipi/digital_fabric/executive.rb +100 -0
data/lib/tipi/digital_fabric/executive/index.html +69 -0
data/lib/tipi/digital_fabric/protocol.rb +90 -0
data/lib/tipi/digital_fabric/request_adapter.rb +48 -0
data/lib/tipi/digital_fabric/service.rb +230 -0
data/lib/tipi/http1_adapter.rb +59 -37
data/lib/tipi/http2_adapter.rb +5 -3
data/lib/tipi/http2_stream.rb +19 -7
data/lib/tipi/rack_adapter.rb +11 -3
data/lib/tipi/version.rb +1 -1
data/lib/tipi/websocket.rb +33 -13
data/test/helper.rb +1 -2
data/test/test_http_server.rb +3 -2
data/test/test_request.rb +108 -0
data/tipi.gemspec +7 -3
metadata +59 -7
data/lib/tipi/request.rb +0 -118

data/examples/http_ws_server.rb CHANGED Viewed

@@ -5,14 +5,14 @@ require 'tipi'
 require 'tipi/websocket'
 def ws_handler(conn)
-  timer = spin do
-    throttled_loop(1) do
-      conn << Time.now.to_s
-    end
+  timer = spin_loop(interval: 1) do
+    conn << Time.now.to_s
   end
   while (msg = conn.recv)
     conn << "you said: #{msg}"
   end
+rescue Exception => e
+  p e
 ensure
   timer.stop
 end
@@ -21,17 +21,15 @@ opts = {
   reuse_addr:  true,
   dont_linger: true,
   upgrade:     {
-    websocket: Polyphony::Websocket.handler(&method(:ws_handler))
+    websocket: Tipi::Websocket.handler(&method(:ws_handler))
   }
 }
 HTML = IO.read(File.join(__dir__, 'ws_page.html'))
-spin do
-  Tipi.serve('0.0.0.0', 1234, opts) do |req|
-    req.respond(HTML, 'Content-Type' => 'text/html')
-  end
-end
 puts "pid: #{Process.pid}"
-puts 'Listening on port 1234...'
+puts 'Listening on port 4411...'
+Tipi.serve('0.0.0.0', 4411, opts) do |req|
+  req.respond(HTML, 'Content-Type' => 'text/html')
+end

data/examples/routing_server.rb ADDED Viewed

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+require 'bundler/setup'
+require 'tipi'
+opts = {
+  reuse_addr:  true,
+  dont_linger: true
+}
+puts "pid: #{Process.pid}"
+puts 'Listening on port 4411...'
+app = Tipi.route do |r|
+  r.root do
+    r.redirect '/hello'
+  end
+  r.on 'hello' do
+    r.get 'world' do
+      r.respond 'Hello world'
+    end
+    r.get do
+      r.respond 'Hello'
+    end
+    r.post do
+      puts 'Someone said Hello'
+      r.redirect '/'
+    end
+  end
+end
+spin do
+  Tipi.serve('0.0.0.0', 4411, opts, &app)
+end.await

data/examples/websocket_client.rb CHANGED Viewed

@@ -26,8 +26,7 @@ class WebsocketClient
   end
   def receive
-    loop do
-      data = @socket.readpartial(8192)
+    @socket.read_loop do |data|
       @reader << data
       parsed = @reader.next
       return parsed if parsed

data/examples/websocket_demo.rb CHANGED Viewed

@@ -30,8 +30,10 @@ class WebsocketClient
   end
   def receive
-    loop do
-      data = @socket.readpartial(8192)
+    parsed = @reader.next
+    return parsed if parsed
+    @socket.read_loop do |data|
       @reader << data
       parsed = @reader.next
       return parsed if parsed

data/examples/ws_page.html CHANGED Viewed

@@ -6,14 +6,13 @@
 <body>
   <script>
     var connect = function () {
-      var exampleSocket = new WebSocket("ws://localhost:1234");
+      var exampleSocket = new WebSocket("wss://dev.realiteq.net/");
       exampleSocket.onopen = function (event) {
         document.querySelector('#status').innerText = 'connected';
         exampleSocket.send("Can you hear me?");
       };
       exampleSocket.onclose = function (event) {
-        console.log('onclose');
         document.querySelector('#status').innerText = 'disconnected';
         setTimeout(function () {
           // exampleSocket.removeAllListeners();

data/lib/tipi.rb CHANGED Viewed

@@ -28,10 +28,8 @@ module Tipi
     end
     def accept_loop(server, opts, &handler)
-      loop do
-        client = server.accept
+      server.accept_loop do |client|
         spin { client_loop(client, opts, &handler) }
-        snooze
       rescue OpenSSL::SSL::SSLError
         # disregard
       end
@@ -42,14 +40,18 @@ module Tipi
       adapter = protocol_adapter(client, opts)
       adapter.each(&handler)
     ensure
-      client.close
+      client.close rescue nil
     end
     def protocol_adapter(socket, opts)
       use_http2 = socket.respond_to?(:alpn_protocol) &&
-      socket.alpn_protocol == H2_PROTOCOL
+                  socket.alpn_protocol == H2_PROTOCOL
       klass = use_http2 ? HTTP2Adapter : HTTP1Adapter
       klass.new(socket, opts)
     end
+    def route(&block)
+      proc { |req| req.route(&block) }
+    end
   end
 end

data/lib/tipi/config_dsl.rb ADDED Viewed

@@ -0,0 +1,153 @@
+# frozen_string_literal: true
+module Tipi
+  module Configuration
+    class Interpreter
+      # make_blank_slate
+      def initialize(assembler)
+        @assembler = assembler
+      end
+      def gzip_response
+        @assembler.emit 'req = Tipi::GZip.wrap(req)'
+      end
+      def log(out)
+        @assembler.wrap_current_frame 'logger.log_request(req) do |req|'
+      end
+      def error(&block)
+        assembler.emit_exception_handler &block
+      end
+      def match(pattern, &block)
+        @assembler.emit_conditional "if req.path =~ #{pattern.inspect}", &block
+      end
+    end
+    class Assembler
+      def self.from_source(code)
+        new.from_source code
+      end
+      def from_source(code)
+        @stack = [new_frame]
+        @app_procs = {}
+        @interpreter = Interpreter.new self
+        @interpreter.instance_eval code
+        loop do
+          frame = @stack.pop
+          return assemble_app_proc(frame).join("\n") if @stack.empty?
+          @stack.last[:body] << assemble_frame(frame)
+        end
+      end
+      def new_frame
+        {
+          prelude: [],
+          body: []
+        }
+      end
+      def add_frame(&block)
+        @stack.push new_frame
+        yield
+      ensure
+        frame = @stack.pop
+        emit assemble(frame)
+      end
+      def wrap_current_frame(head)
+        frame = @stack.pop
+        wrapper = new_frame
+        wrapper[:body] << head
+        @stack.push wrapper
+        @stack.push frame
+      end
+      def emit(code)
+        @stack.last[:body] << code
+      end
+      def emit_prelude(code)
+        @stack.last[:prelude] << code
+      end
+      def emit_exception_handler(&block)
+        proc_id = add_app_proc block
+        @stack.last[:rescue_proc_id] = proc_id
+      end
+      def emit_block(conditional, &block)
+        proc_id = add_app_proc block
+        @stack.last[:branched] = true
+        emit conditional
+        add_frame &block
+      end
+      def add_app_proc(proc)
+        id = :"proc#{@app_procs.size}"
+        @app_procs[id] = proc
+        id
+      end
+      def assemble_frame(frame)
+        indent = 0
+        lines = []
+        emit_code lines, frame[:prelude], indent
+        if frame[:rescue_proc_id]
+          emit_code lines, 'begin', indent
+          indent += 1
+        end
+        emit_code lines, frame[:body], indent
+        if frame[:rescue_proc_id]
+          emit_code lines, 'rescue => e', indent
+          emit_code lines, "  app_procs[#{frame[:rescue_proc_id].inspect}].call(req, e)", indent
+          emit_code lines, 'end', indent
+          indent -= 1
+        end
+        lines
+      end
+      def assemble_app_proc(frame)
+        indent = 0
+        lines = []
+        emit_code lines, frame[:prelude], indent
+        emit_code lines, 'proc do |req|', indent
+        emit_code lines, frame[:body], indent + 1
+        emit_code lines, 'end', indent
+        lines
+      end
+      def emit_code(lines, code, indent)
+        if code.is_a? Array
+          code.each { |l| emit_code lines, l, indent + 1 }
+        else
+          lines << (indent_line code, indent)
+        end
+      end
+      @@indents = Hash.new { |h, k| h[k] =  '  ' * k }
+      def indent_line(code, indent)
+        indent == 0 ? code : "#{ @@indents[indent] }#{code}"
+      end
+    end
+  end
+end
+def assemble(code)
+  Tipi::Configuration::Assembler.from_source(code)
+end
+code = assemble <<~RUBY
+gzip_response
+log STDOUT
+RUBY
+puts code

data/lib/tipi/configuration.rb CHANGED Viewed

@@ -19,7 +19,7 @@ module Tipi
       end
       def start_listeners(config)
-        puts "listening on port 1234"
+        puts "Listening on port 1234"
         @server = Polyphony::Net.tcp_listen('0.0.0.0', 1234, { reuse_addr: true, dont_linger: true })
       end

data/lib/tipi/digital_fabric.rb ADDED Viewed

@@ -0,0 +1,7 @@
+module DigitalFabric
+end
+::DF = DigitalFabric
+require_relative 'digital_fabric/service'
+require_relative 'digital_fabric/agent_proxy'

data/lib/tipi/digital_fabric/agent.rb ADDED Viewed

@@ -0,0 +1,225 @@
+# frozen_string_literal: true
+require_relative './protocol'
+require_relative './request_adapter'
+require 'msgpack'
+require 'tipi/websocket'
+require 'tipi/request'
+module DigitalFabric
+  class Agent
+    def initialize(server_url, route, token)
+      @server_url = server_url
+      @route = route
+      @token = token
+      @requests = {}
+      @long_running_requests = {}
+      @name = '<unknown>'
+    end
+    class TimeoutError < RuntimeError
+    end
+    class GracefulShutdown < RuntimeError
+    end
+    def run
+      @fiber = Fiber.current
+      @keep_alive_timer = spin_loop(interval: 5) { keep_alive }
+      while true
+        connect_and_process_incoming_requests
+        return if @shutdown
+        sleep 5
+      end
+    ensure
+      @keep_alive_timer.stop
+    end
+    def connect_and_process_incoming_requests
+      # log 'Connecting...'
+      @socket = connect_to_server
+      @last_recv = @last_send = Time.now
+      df_upgrade
+      @connected = true
+      @msgpack_reader = MessagePack::Unpacker.new
+      process_incoming_requests
+    rescue IOError, Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EPIPE, TimeoutError
+      log 'Disconnected' if @connected
+      @connected = nil
+    end
+    def connect_to_server
+      if @server_url =~ /^([^\:]+)\:(\d+)$/
+        host = Regexp.last_match(1)
+        port = Regexp.last_match(2)
+        Polyphony::Net.tcp_connect(host, port)
+      else
+        UNIXSocket.new(@server_url)
+      end
+    end
+    UPGRADE_REQUEST = <<~HTTP
+    GET / HTTP/1.1
+    Host: localhost
+    Connection: upgrade
+    Upgrade: df
+    DF-Token: %s
+    DF-Mount: %s
+    HTTP
+    def df_upgrade
+      @socket << format(UPGRADE_REQUEST, @token, mount_point)
+      while (line = @socket.gets)
+        break if line.chomp.empty?
+      end
+      # log 'Connection upgraded'
+    end
+    def mount_point
+      if @route[:host]
+        "host=#{@route[:host]}"
+      elsif @route[:path]
+        "path=#{@route[:path]}"
+      else
+        nil
+      end
+    end
+    def log(msg)
+      puts "#{Time.now} (#{@name}) #{msg}"
+    end
+    def process_incoming_requests
+      @socket.feed_loop(@msgpack_reader, :feed_each) do |msg|
+        recv_df_message(msg)
+        return if @shutdown && @requests.empty?
+      end
+    rescue IOError, SystemCallError, TimeoutError
+      # ignore
+    end
+    def keep_alive
+      return unless @connected
+      now = Time.now
+      if now - @last_send >= Protocol::SEND_TIMEOUT
+        send_df_message(Protocol.ping)
+      end
+      # if now - @last_recv >= Protocol::RECV_TIMEOUT
+      #   raise TimeoutError
+      # end
+    rescue IOError, SystemCallError => e
+      # transmit exception to fiber running the agent
+      @fiber.raise(e)
+    end
+    def recv_df_message(msg)
+      @last_recv = Time.now
+      case msg['kind']
+      when Protocol::SHUTDOWN
+        recv_shutdown
+      when Protocol::HTTP_REQUEST
+        recv_http_request(msg)
+      when Protocol::HTTP_REQUEST_BODY
+        recv_http_request_body(msg)
+      when Protocol::WS_REQUEST
+        recv_ws_request(msg)
+      when Protocol::CONN_DATA, Protocol::CONN_CLOSE,
+           Protocol::WS_DATA, Protocol::WS_CLOSE
+        fiber = @requests[msg['id']]
+        fiber << msg if fiber
+      end
+    end
+    def send_df_message(msg)
+      # we mark long-running requests by applying simple heuristics to sent DF
+      # messages. This is so we can correctly stop long-running requests
+      # upon graceful shutdown
+      if is_long_running_request_response?(msg)
+        id = msg[:id]
+        @long_running_requests[id] = @requests[id]
+      end
+      @last_send = Time.now
+      @socket << msg.to_msgpack
+    end
+    def is_long_running_request_response?(msg)
+      case msg[:kind]
+      when Protocol::HTTP_UPGRADE
+        true
+      when Protocol::HTTP_RESPONSE
+        msg[:body] && !msg[:complete]
+      end
+    end
+    def recv_shutdown
+      # puts "Received shutdown message (#{@requests.size} pending requests)"
+      # puts "  (Long running requests: #{@long_running_requests.size})"
+      @shutdown = true
+      @long_running_requests.values.each { |f| f.terminate(true) }
+    end
+    def recv_http_request(msg)
+      req = prepare_http_request(msg)
+      id = msg['id']
+      @requests[id] = spin do
+        http_request(req)
+      rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
+        # ignore
+      rescue Polyphony::Terminate
+        req.respond(nil, { ':status' => Qeweney::Status::SERVICE_UNAVAILABLE }) if Fiber.current.graceful_shutdown?
+      ensure
+        @requests.delete(id)
+        @long_running_requests.delete(id)
+        @fiber.terminate if @shutdown && @requests.empty?
+      end
+    end
+    def prepare_http_request(msg)
+      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
+      req.buffer_body_chunk(msg['body']) if msg['body']
+      req.complete! if msg['complete']
+      req
+    end
+    def recv_http_request_body(msg)
+      fiber = @requests[msg['id']]
+      return unless fiber
+      fiber << msg['body']
+    end
+    def get_http_request_body(id, limit)
+      send_df_message(Protocol.http_get_request_body(id, limit))
+      receive
+    end
+    def recv_ws_request(msg)
+      req = Qeweney::Request.new(msg['headers'], RequestAdapter.new(self, msg))
+      id = msg['id']
+      @requests[id] = @long_running_requests[id] = spin do
+        ws_request(req)
+      rescue IOError, Errno::ECONNREFUSED, Errno::EPIPE
+        # ignore
+      ensure
+        @requests.delete(id)
+        @long_running_requests.delete(id)
+        @fiber.terminate if @shutdown && @requests.empty?
+      end
+    end
+    # default handler for HTTP request
+    def http_request(req)
+      req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
+    end
+    # default handler for WS request
+    def ws_request(req)
+      req.respond(nil, { ':status': Qeweney::Status::SERVICE_UNAVAILABLE })
+    end
+  end
+end