tipi 0.32 → 0.37

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0fd31b18e0c98a8beb780933f7adef48e0e4403a765cc64f37ef02515d94723
-  data.tar.gz: 1cc72a408cd8fe9b4d1528bd7b1200f3e7a71098596f5b18812d2d75b6120936
+  metadata.gz: 4129b10a7f6b5fb92e4e4784bf8cfd12ea2659840b616b7627bf28d1dc423e87
+  data.tar.gz: 7f5f71c4a870e33c7de84fd292568b3aca582dcc1adc9496d99e636328b3df4b
 SHA512:
-  metadata.gz: d4fb34dbec4798561322546a90f8104f3e91bd806923ba6d73e3267c06ab05499104431bf15f1abb0984d014626a062267305370470d80a7c61776a2f0c0bd8b
-  data.tar.gz: b0b7d9ec9e0536acef6f9675d592d7c48123bf591468326178b96ef673f194c44a43dcd57ed7f860d78acfe27f71192378f328b5531efff73731cd4b3875c3ee
+  metadata.gz: c1bfcbf1cdf4fa2868554e690a71543d89a37bf2d8cfa27cf604e453fb411d5b1dd2e4bc10678cbe79db6a94238ffef1a09e53771f7e8d8e71f2512b87f7265a
+  data.tar.gz: 4f476a65800ffb6cbcdfde6e10ed9e1dd7b9fcc4ddd69ad2babed1a747cba4f94e05659cbd235c1f7e71e1595851d77896262d45d1036d216d866b7bdc3f914a

data/CHANGELOG.md

@@ -1,3 +1,30 @@
+## 0.37 2021-02-15
+
+* Update upgrade mechanism to work with updated Qeweney API
+
+## 0.36 2021-02-12
+
+* Use `Qeweney::Status` constants
+
+## 0.35 2021-02-10
+
+* Extract Request class into separate [qeweney](https://github.com/digital-fabric/qeweney) gem
+
+## 0.34 2021-02-07
+
+* Implement digital fabric service and agents
+* Add multipart and urlencoded form data parsing
+* Improve request body reading behaviour
+* Add more `Request` information methods
+* Add access to connection for HTTP2 requests
+* Allow calling `Request#send_chunk` with empty chunk
+* Add support for handling protocol upgrades from within request handler
+
+## 0.33 2020-11-20
+
+* Update code for Polyphony 0.47.5
+* Add support for Rack::File body to Tipi::RackAdapter
+
 ## 0.32 2020-08-14
 
 * Respond with array of strings instead of concatenating for HTTP 1

data/Gemfile.lock

@@ -1,10 +1,12 @@
 PATH
   remote: .
   specs:
-    tipi (0.32)
+    tipi (0.37)
       http-2 (~> 0.10.0)
       http_parser.rb (~> 0.6.0)
-      polyphony (~> 0.45)
+      msgpack (~> 1.4.2)
+      polyphony (~> 0.51.0)
+      qeweney (~> 0.5)
       rack (>= 2.0.8, < 2.3.0)
       websocket (~> 1.2.8)
 
@@ -14,6 +16,7 @@ GEM
     ansi (1.5.0)
     builder (3.2.4)
     docile (1.3.2)
+    escape_utils (1.2.1)
     http-2 (0.10.2)
     http_parser.rb (0.6.0)
     json (2.3.1)
@@ -24,7 +27,10 @@ GEM
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    polyphony (0.45.2)
+    msgpack (1.4.2)
+    polyphony (0.51.0)
+    qeweney (0.5)
+      escape_utils (~> 1.2.1)
     rack (2.2.3)
     rake (12.3.3)
     ruby-progressbar (1.10.1)
@@ -33,7 +39,7 @@ GEM
       json (>= 1.8, < 3)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.2)
-    websocket (1.2.8)
+    websocket (1.2.9)
 
 PLATFORMS
   ruby

data/LICENSE

@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2018 Sharon Rosner
+Copyright (c) 2021 Sharon Rosner
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

data/TODO.md

@@ -1,10 +1,20 @@
+# Digital Fabric
+
+Problems to fix:
+
+- Memory leak (in server? multi agent? multi client?)
+
 # Roadmap
 
 - Update README (get rid of non-http stuff)
 - Improve Rack spec compliance, add tests
-- Homogenize HTTP 1 and HTTP 2 headers - upcase ? downcase ?
+- Homogenize HTTP 1 and HTTP 2 headers - downcase symbols
 
-## 0.30
+- Use `http-2-next` instead of `http-2` for http/2
+  - https://gitlab.com/honeyryderchuck/http-2-next
+  - Open an issue there, ask what's the difference between the two gems?
+
+## 0.35
 
 - Add more poly CLI commands and options:
 
@@ -16,51 +26,7 @@
   - set port to bind to
   - set forking process count
 
-## 0.31 Working Sinatra application
+## 0.36 Working Sinatra application
 
 - app with database access (postgresql)
 - benchmarks!
-
-# HTTP Client Agent
-
-The concurrency model and the fact that we want to serve the response object on
-receiving headers and let the user lazily read the response body, means we'll
-need to change the API to accept a block:
-
-```ruby
-# current API
-resp = Agent.get('http://acme.org')
-puts resp.body
-
-# proposed API
-Agent.get('http://acme.org') do |resp|
-  puts resp.body
-end
-```
-
-While the block is running, the connection adapter is acquired. Once the block
-is done running, the request (and response) can be discarded. The problem with
-that if we spin up a coprocess from that block we risk all kinds of race
-conditions and weird behaviours.
-
-A compromise might be to allow the two: doing a `get` without providing a block
-will return a response object that already has the body (i.e. the entire
-response has already been received). Doing a `get` with a block will invoke the
-block once headers are received, letting the user's code stream the body:
-
-```ruby
-def request(ctx, &block)
-  ...
-  connection_manager.acquire do |adapter|
-    response = adapter.request(ctx)
-    if block
-      block.(response)
-    else
-      # wait for body
-      response.body
-    end
-    response
-  end
-end
-```
-

data/bin/tipi

@@ -7,7 +7,20 @@ require File.expand_path('../lib/tipi/configuration', __dir__)
 config = {}
 #config[:forked] = 4
 
+puts DATA.read
+puts
+
 configuration_manager = spin { Tipi::Configuration.supervise_config }
 
 configuration_manager << config
 configuration_manager.await
+
+__END__
+
+          ooo
+       oo
+     o
+   \|/
+   / \   Tipi - A better web server for a better world
+  /___\
+  

data/df/agent.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'json'
+require 'tipi/digital_fabric/protocol'
+require 'tipi/digital_fabric/agent'
+
+Protocol = DigitalFabric::Protocol
+
+class SampleAgent < DigitalFabric::Agent
+  def initialize(id, server_url)
+    @id = id
+    super(server_url, { host: "#{id}.realiteq.net" }, 'foobar')
+    @name = "agent-#{@id}"
+  end
+
+  def http_request(req)
+    return streaming_http_request(req) if req.path == '/streaming'
+    return form_http_request(req) if req.path == '/form'
+
+    req.respond({ id: @id, time: Time.now.to_i }.to_json)
+  end
+
+  def streaming_http_request(req)
+    req.send_headers({ 'Content-Type': 'text/json' })
+
+    60.times do
+      sleep 1
+      do_some_activity
+      req.send_chunk({ id: @id, time: Time.now.to_i }.to_json)
+    end
+  
+    req.finish
+  rescue Polyphony::Terminate
+    req.respond(' * shutting down *') if Fiber.current.graceful_shutdown?
+  rescue Exception => e
+    p e
+    puts e.backtrace.join("\n")
+  end
+
+  def form_http_request(req)
+    body = req.read
+    form_data = Tipi::Request.parse_form_data(body, req.headers)
+    req.respond({ form_data: form_data, headers: req.headers }.to_json, { 'Content-Type': 'text/json' })
+  end
+
+  def do_some_activity
+    File.open('/tmp/df-test.log', 'a+') { |f| sleep rand; f.puts "#{Time.now} #{@name} #{generate_data(2**8)}" }
+  end
+
+  def generate_data(length)
+    charset = Array('A'..'Z') + Array('a'..'z') + Array('0'..'9')
+    Array.new(length) { charset.sample }.join
+  end
+end
+
+# id = ARGV[0]
+# puts "Starting agent #{id} pid: #{Process.pid}"
+
+# spin_loop(interval: 60) { GC.start }
+# SampleAgent.new(id, '/tmp/df.sock').run
+# SampleAgent.new(id, 'localhost:4411').run

data/df/etc_benchmark.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+def generate
+  SecureRandom.uuid
+end
+
+count = 100000
+
+GC.disable
+t0 = Time.now
+count.times { generate }
+elapsed = Time.now - t0
+puts "rate: #{count / elapsed}/s"

data/df/multi_agent_supervisor.rb

@@ -0,0 +1,87 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'json'
+
+require 'fileutils'
+FileUtils.cd(__dir__)
+
+require_relative 'agent'
+
+class AgentManager
+  def initialize
+    @running_agents = {}
+    @pending_actions = Queue.new
+    @processor = spin_loop { process_pending_action }
+  end
+
+  def process_pending_action
+    action = @pending_actions.shift
+    case action[:kind]
+    when :start
+      start_agent(action[:spec])
+    when :stop
+      stop_agent(action[:spec])
+    end
+  end
+
+  def start_agent(spec)
+    return if @running_agents[spec]
+
+    @running_agents[spec] = spin do
+      while true
+        launch_agent_from_spec(spec)
+        sleep 1
+      end
+    ensure
+      @running_agents.delete(spec)
+    end
+  end
+
+  def stop_agent(spec)
+    fiber = @running_agents[spec]
+    return unless fiber
+
+    fiber.terminate
+    fiber.await
+  end
+
+  def update
+    return unless @pending_actions.empty?
+
+    current_specs = @running_agents.keys
+    updated_specs = agent_specs
+
+    to_start = updated_specs - current_specs
+    to_stop = current_specs - current_specs
+
+    to_start.each { |s| @pending_actions << { kind: :start, spec: s } }
+    to_stop.each { |s| @pending_actions << { kind: :stop, spec: s } }
+  end
+
+  def run
+    every(2) { update }
+  end
+end
+
+class RealityAgentManager < AgentManager
+  def agent_specs
+    (1..400).map { |i| { id: i } }
+  end
+
+  def launch_agent_from_spec(spec)
+    # Polyphony::Process.watch("ruby agent.rb #{spec[:id]}")
+    Polyphony::Process.watch do
+      spin_loop(interval: 60) { GC.start }
+      agent = SampleAgent.new(spec[:id], '/tmp/df.sock')
+      puts "Starting agent #{spec[:id]} pid: #{Process.pid}"
+      agent.run
+    end
+  end
+end
+
+puts "Agent manager pid: #{Process.pid}"
+
+manager = RealityAgentManager.new
+manager.run

data/df/multi_client.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require 'bundler/setup'
+require 'polyphony'
+require 'http/parser'
+
+class Client
+  def initialize(id, host, port, http_host, interval)
+    @id = id
+    @host = host
+    @port = port
+    @http_host = http_host
+    @interval = interval.to_f
+    @interval_delta = @interval / 2
+  end
+
+  def run
+    while true
+      connect && issue_requests
+      sleep 5
+    end
+  end
+
+  def connect
+    @socket = Polyphony::Net.tcp_connect(@host, @port)
+  rescue SystemCallError
+    false
+  end
+
+  REQUEST = <<~HTTP
+  GET / HTTP/1.1
+  Host: %s
+
+  HTTP
+
+  def issue_requests
+    @parser = Http::Parser.new
+    @parser.on_message_complete = proc { @got_reply = true }
+    @parser.on_body = proc { |chunk| @response = chunk }
+
+    while true
+      do_request
+      sleep rand((@interval - @interval_delta)..(@interval + @interval_delta))
+    end
+  rescue IOError, Errno::EPIPE, Errno::ECONNRESET, Errno::ECONNREFUSED => e
+    # fail quitely
+  ensure
+    @parser = nil
+  end
+
+  def do_request
+    @got_reply = nil
+    @response = nil
+    @socket << format(REQUEST, @http_host)
+    wait_for_response
+    # if @parser.status_code != 200
+    #   puts "Got status code #{@parser.status_code} from #{@http_host} => #{@parser.headers && @parser.headers['X-Request-ID']}"
+    # end
+    # puts "#{Time.now} [client-#{@id}] #{@http_host} => #{@response || '<error>'}"
+  end
+
+  def wait_for_response
+    @socket.recv_loop do |data|
+      @parser << data
+      return @response if @got_reply
+    end
+  end
+end
+
+def spin_client(id, host)
+  spin do
+    client = Client.new(id, 'localhost', 4411, host, 30)
+    client.run
+  end
+end
+
+spin_loop(interval: 60) { GC.start }
+
+10000.times { |id| spin_client(id, "#{rand(1..400)}.realiteq.net") }
+
+trap('SIGINT') { exit! }
+
+puts "Multi client pid: #{Process.pid}"
+sleep