tipi 0.33 → 0.37.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: adbbb6098994749bc96df90a6c73820ca54245c6026141768c83d883677fd6d3
-  data.tar.gz: daf182efaff0561bb6ff85706ae14ab70de68f840290eb5bb88e870fc0a29199
+  metadata.gz: 46b961262c4d31ce6e479864bced5b698f77e757f60e93a9dff8ba7401f2fa0d
+  data.tar.gz: 23881ff338c0b7aeaea31a7700ed08271eca7349322fb1868246c1541386c2b8
 SHA512:
-  metadata.gz: 78c8fb0c465779673563fb87857fdc935c4906830086f664657c00b2b80686f4fa35c3673a554976d26a41af072686ad2c197dd1fbd6a14d875480831e8455ff
-  data.tar.gz: bbb9c98e7f167aae1a006fd30ac7557ddf9671d3951e00012582b748f3afea363e53b166c40ec1af3be0f5c6d96d1991f45d9e549f5fd23e614345508719ef5f
+  metadata.gz: ccb684d646130d74292809c1c3080f5c4280d260ae38da119813d9137e84278dfe61cf7471512353b95ea36ced2bb2913fd23695e1b3ce549a17a66d243c1bd6
+  data.tar.gz: fe7e6ed4c6b10ae4d409edbf9e3a105d49763cccc6c8ed81f42af5b7a903b45f53548bff830a63051c5626cd107fe169538bba08b83f8a26adf5b36f51a4f070

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.37 2021-02-15
+
+* Update upgrade mechanism to work with updated Qeweney API
+
+## 0.36 2021-02-12
+
+* Use `Qeweney::Status` constants
+
+## 0.35 2021-02-10
+
+* Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
+
+## 0.34 2021-02-07
+
+* Implement digital fabric service and agents
+* Add multipart and urlencoded form data parsing
+* Improve request body reading behaviour
+* Add more `Request` information methods
+* Add access to connection for HTTP2 requests
+* Allow calling `Request#send_chunk` with empty chunk
+* Add support for handling protocol upgrades from within request handler
+
 ## 0.33 2020-11-20
 
 * Update code for Polyphony 0.47.5

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    tipi (0.33)
+    tipi (0.37.1)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
-      polyphony (~> 0.47.5.1)
+      msgpack (~> 1.4.2)
+      polyphony (~> 0.52.0)
+      qeweney (~> 0.6)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -14,6 +16,7 @@ GEM
     ansi (1.5.0)
     builder (3.2.4)
     docile (1.3.2)
+    escape_utils (1.2.1)
     http-2 (0.10.2)
     http_parser.rb (0.6.0)
     json (2.3.1)
@@ -24,7 +27,10 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    polyphony (0.47.5.1)
+    msgpack (1.4.2)
+    polyphony (0.52.0)
+    qeweney (0.6)
+      escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)
     ruby-progressbar (1.10.1)
@@ -33,7 +39,7 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    websocket (1.2.8)
+    websocket (1.2.9)
 
 PLATFORMS
   ruby

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Sharon Rosner
+Copyright (c) 2021 Sharon Rosner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/TODO.md

@@ -1,10 +1,18 @@
+For immediate execution:
+
+
+
 # Roadmap
 
 - Update README (get rid of non-http stuff)
 - Improve Rack spec compliance, add tests
-- Homogenize HTTP 1 and HTTP 2 headers - upcase ? downcase ?
+- Homogenize HTTP 1 and HTTP 2 headers - downcase symbols
 
-## 0.30
+- Use `http-2-next` instead of `http-2` for http/2
+  - https://gitlab.com/honeyryderchuck/http-2-next
+  - Open an issue there, ask what's the difference between the two gems?
+
+## 0.38
 
 - Add more poly CLI commands and options:
 
@@ -16,51 +24,7 @@
   - set port to bind to
   - set forking process count
 
-## 0.31 Working Sinatra application
+## 0.39 Working Sinatra application
 
 - app with database access (postgresql)
 - benchmarks!
-
-# HTTP Client Agent
-
-The concurrency model and the fact that we want to serve the response object on
-receiving headers and let the user lazily read the response body, means we'll
-need to change the API to accept a block:
-
-```ruby
-# current API
-resp = Agent.get('http://acme.org')
-puts resp.body
-
-# proposed API
-Agent.get('http://acme.org') do |resp|
-  puts resp.body
-end
-```
-
-While the block is running, the connection adapter is acquired. Once the block
-is done running, the request (and response) can be discarded. The problem with
-that if we spin up a coprocess from that block we risk all kinds of race
-conditions and weird behaviours.
-
-A compromise might be to allow the two: doing a `get` without providing a block
-will return a response object that already has the body (i.e. the entire
-response has already been received). Doing a `get` with a block will invoke the
-block once headers are received, letting the user's code stream the body:
-
-```ruby
-def request(ctx, &block)
-  ...
-  connection_manager.acquire do |adapter|
-    response = adapter.request(ctx)
-    if block
-      block.(response)
-    else
-      # wait for body
-      response.body
-    end
-    response
-  end
-end
-```
-

data/df/agent.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'json'
+require 'tipi/digital_fabric/protocol'
+require 'tipi/digital_fabric/agent'
+
+Protocol = DigitalFabric::Protocol
+
+class SampleAgent < DigitalFabric::Agent
+  def initialize(id, server_url)
+    @id = id
+    super(server_url, { host: "#{id}.realiteq.net" }, 'foobar')
+    @name = "agent-#{@id}"
+  end
+
+  def http_request(req)
+    return streaming_http_request(req) if req.path == '/streaming'
+    return form_http_request(req) if req.path == '/form'
+
+    req.respond({ id: @id, time: Time.now.to_i }.to_json)
+  end
+
+  def streaming_http_request(req)
+    req.send_headers({ 'Content-Type': 'text/json' })
+
+    60.times do
+      sleep 1
+      do_some_activity
+      req.send_chunk({ id: @id, time: Time.now.to_i }.to_json)
+    end
+  
+    req.finish
+  rescue Polyphony::Terminate
+    req.respond(' * shutting down *') if Fiber.current.graceful_shutdown?
+  rescue Exception => e
+    p e
+    puts e.backtrace.join("\n")
+  end
+
+  def form_http_request(req)
+    body = req.read
+    form_data = Tipi::Request.parse_form_data(body, req.headers)
+    req.respond({ form_data: form_data, headers: req.headers }.to_json, { 'Content-Type': 'text/json' })
+  end
+
+  def do_some_activity
+    File.open('/tmp/df-test.log', 'a+') { |f| sleep rand; f.puts "#{Time.now} #{@name} #{generate_data(2**8)}" }
+  end
+
+  def generate_data(length)
+    charset = Array('A'..'Z') + Array('a'..'z') + Array('0'..'9')
+    Array.new(length) { charset.sample }.join
+  end
+end
+
+# id = ARGV[0]
+# puts "Starting agent #{id} pid: #{Process.pid}"
+
+# spin_loop(interval: 60) { GC.start }
+# SampleAgent.new(id, '/tmp/df.sock').run
+# SampleAgent.new(id, 'localhost:4411').run

data/df/etc_benchmark.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+def generate
+  SecureRandom.uuid
+end
+
+count = 100000
+
+GC.disable
+t0 = Time.now
+count.times { generate }
+elapsed = Time.now - t0
+puts "rate: #{count / elapsed}/s"

data/df/multi_agent_supervisor.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'json'
+
+require 'fileutils'
+FileUtils.cd(__dir__)
+
+require_relative 'agent'
+
+class AgentManager
+  def initialize
+    @running_agents = {}
+    @pending_actions = Queue.new
+    @processor = spin_loop { process_pending_action }
+  end
+
+  def process_pending_action
+    action = @pending_actions.shift
+    case action[:kind]
+    when :start
+      start_agent(action[:spec])
+    when :stop
+      stop_agent(action[:spec])
+    end
+  end
+
+  def start_agent(spec)
+    return if @running_agents[spec]
+
+    @running_agents[spec] = spin do
+      while true
+        launch_agent_from_spec(spec)
+        sleep 1
+      end
+    ensure
+      @running_agents.delete(spec)
+    end
+  end
+
+  def stop_agent(spec)
+    fiber = @running_agents[spec]
+    return unless fiber
+
+    fiber.terminate
+    fiber.await
+  end
+
+  def update
+    return unless @pending_actions.empty?
+
+    current_specs = @running_agents.keys
+    updated_specs = agent_specs
+
+    to_start = updated_specs - current_specs
+    to_stop = current_specs - current_specs
+
+    to_start.each { |s| @pending_actions << { kind: :start, spec: s } }
+    to_stop.each { |s| @pending_actions << { kind: :stop, spec: s } }
+  end
+
+  def run
+    every(2) { update }
+  end
+end
+
+class RealityAgentManager < AgentManager
+  def agent_specs
+    (1..400).map { |i| { id: i } }
+  end
+
+  def launch_agent_from_spec(spec)
+    # Polyphony::Process.watch("ruby agent.rb #{spec[:id]}")
+    Polyphony::Process.watch do
+      spin_loop(interval: 60) { GC.start }
+      agent = SampleAgent.new(spec[:id], '/tmp/df.sock')
+      puts "Starting agent #{spec[:id]} pid: #{Process.pid}"
+      agent.run
+    end
+  end
+end
+
+puts "Agent manager pid: #{Process.pid}"
+
+manager = RealityAgentManager.new
+manager.run

data/df/multi_client.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'http/parser'
+
+class Client
+  def initialize(id, host, port, http_host, interval)
+    @id = id
+    @host = host
+    @port = port
+    @http_host = http_host
+    @interval = interval.to_f
+    @interval_delta = @interval / 2
+  end
+
+  def run
+    while true
+      connect && issue_requests
+      sleep 5
+    end
+  end
+
+  def connect
+    @socket = Polyphony::Net.tcp_connect(@host, @port)
+  rescue SystemCallError
+    false
+  end
+
+  REQUEST = <<~HTTP
+  GET / HTTP/1.1
+  Host: %s
+
+  HTTP
+
+  def issue_requests
+    @parser = Http::Parser.new
+    @parser.on_message_complete = proc { @got_reply = true }
+    @parser.on_body = proc { |chunk| @response = chunk }
+
+    while true
+      do_request
+      sleep rand((@interval - @interval_delta)..(@interval + @interval_delta))
+    end
+  rescue IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED => e
+    # fail quitely
+  ensure
+    @parser = nil
+  end
+
+  def do_request
+    @got_reply = nil
+    @response = nil
+    @socket << format(REQUEST, @http_host)
+    wait_for_response
+    # if @parser.status_code != 200
+    #   puts "Got status code #{@parser.status_code} from #{@http_host} => #{@parser.headers && @parser.headers['X-Request-ID']}"
+    # end
+    # puts "#{Time.now} [client-#{@id}] #{@http_host} => #{@response || '<error>'}"
+  end
+
+  def wait_for_response
+    @socket.recv_loop do |data|
+      @parser << data
+      return @response if @got_reply
+    end
+  end
+end
+
+def spin_client(id, host)
+  spin do
+    client = Client.new(id, 'localhost', 4411, host, 30)
+    client.run
+  end
+end
+
+spin_loop(interval: 60) { GC.start }
+
+10000.times { |id| spin_client(id, "#{rand(1..400)}.realiteq.net") }
+
+trap('SIGINT') { exit! }
+
+puts "Multi client pid: #{Process.pid}"
+sleep

data/df/routing_benchmark.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'tipi/digital_fabric'
+
+class FakeAgent
+  def initialize(idx)
+    @idx = idx
+  end
+end
+
+def setup_df_service_with_agents(agent_count)
+  server = DigitalFabric::Service.new
+  agent_count.times do |i|
+    server.mount({path: "/#{i}"}, FakeAgent.new(i))
+  end
+  server
+end
+
+def benchmark_route_compilation(agent_count, iterations)
+  service = setup_df_service_with_agents(agent_count)
+  t0 = Time.now
+  iterations.times { service.compile_agent_routes }
+  elapsed = Time.now - t0
+  puts "route_compilation: #{agent_count} => #{elapsed / iterations}s (#{1/(elapsed / iterations)} ops/sec)"
+end
+
+class FauxRequest
+  def initialize(agent_count)
+    @agent_count = agent_count
+  end
+
+  def headers
+    { ':path' => "/#{rand(@agent_count)}"}
+  end
+end
+
+def benchmark_find_agent(agent_count, iterations)
+  service = setup_df_service_with_agents(agent_count)
+  t0 = Time.now
+  request = FauxRequest.new(agent_count)
+  iterations.times do
+    agent = service.find_agent(request)
+  end
+  elapsed = Time.now - t0
+  puts "routing: #{agent_count} => #{elapsed / iterations}s (#{1/(elapsed / iterations)} ops/sec)"
+end
+
+def benchmark
+  benchmark_route_compilation(100, 1000)
+  benchmark_route_compilation(500,  200)
+  benchmark_route_compilation(1000, 100)
+
+  benchmark_find_agent(100, 1000)
+  benchmark_find_agent(500,  200)
+  benchmark_find_agent(1000, 100)
+end
+
+benchmark