spyder 0.1.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8fbffd16e4740ebf51ca1e51cb49c5a2f665408580340786ac36ba484f045c4
-  data.tar.gz: 6a0c26ce6168805cf1e1636835bd3aa4e376b022be5d4f2874bdfbc80682523e
+  metadata.gz: b5ee9dba2fca697adc1456d6473a6a1f7e011108ce4d8baf0f7659c9886b85c0
+  data.tar.gz: 719a5fe8d53574d442ddec5cbb666db6018526901be76ebf7ac665451da3c512
 SHA512:
-  metadata.gz: 4cac8464742015ecba92eb7d0ff9ca0d3c7bde3ca14ea274200344de1b1fd92aa424aede419ceab17afe6e60693f54ed6077c27ab233459496e03a780026b9b2
-  data.tar.gz: 37b4244fc29821e8efa337d7ffd0cf6e490bf1e94f526933bfafa1dea08b87c30b5312bfd8c6bf8209794108fab82bb0fc9debf9f2c72cc758cc16d9152104fd
+  metadata.gz: 90c3f303a6d4c12abf9250ad13caba9cdf06632cfc4a68a317c7a8ccac47b2aa4c568c315581a03a353d9d0b15317a4dbe1e245ad9a1e70335a2406cb9f30ba3
+  data.tar.gz: 471ab8a863891a2b27f236bb0cea65028e37e62ac12a387daa81a302d58adf5cf7f81c236b8d4953e9e15815a3b56c86a15bf497f7c207a1ac4c8fecfe2a1447

data/lib/spyder/response.rb

@@ -4,6 +4,7 @@ module Spyder
   class Response
     attr_reader :code
     attr_accessor :body
+    attr_accessor :hijack
     attr_reader :headers
     attr_writer :reason_sentence
 
@@ -60,8 +61,10 @@ module Spyder
       ]
     end.freeze
 
-    def initialize
-      self.code = :ok
+    def initialize(code: :ok)
+      @body = nil
+      @hijack = false
+      self.code = code
       @headers = HeaderStore.new(:response)
     end
 

data/lib/spyder/router.rb

@@ -2,10 +2,11 @@
 
 module Spyder
   class Router
-    attr_accessor :fallback_route
+    attr_accessor :fallback_stack
 
     def initialize
       @routes = Hash.new { |k, v| k[v] = [] }
+      @fallback_stack = []
     end
 
     def add_route(verb, matcher, &handler)
@@ -14,6 +15,12 @@ module Spyder
       @routes[verb.to_s.upcase] << [matcher, handler]
     end
 
+    def add_fallback(callable = nil, &blk)
+      raise "Provide either a callable or a block, but not both" if callable && block_given?
+
+      @fallback_stack << (block_given? ? blk : callable)
+    end
+
     def call(ctx, request)
       only_path = request.path_info
 
@@ -26,7 +33,16 @@ module Spyder
         end
       end
 
-      match_data ? handler.call(request, match_data) : Response.make_generic(:not_found)
+      response = match_data ? handler.call(request, match_data) : nil
+      return response if response.is_a?(Response)
+
+      fallback_stack.each do |blk|
+        response = blk.call(request)
+
+        return response if response.is_a?(Response)
+      end
+
+      Response.make_generic(:not_found)
     end
   end
 

data/lib/spyder/server.rb

@@ -4,8 +4,9 @@ module Spyder
   class Server
     attr_accessor :router
 
-    def initialize(bind, port, router: Router.new, max_threads: 4)
+    def initialize(bind, port, router: Router.new, max_threads: 4, tcp_backlog: 10)
       @server = TCPServer.new(bind, port)
+      @tcp_backlog = tcp_backlog
       @max_threads = max_threads
       @middleware = []
       @threads = []
@@ -13,49 +14,54 @@ module Spyder
       @router = router
     end
 
-    def add_middleware(callable, args)
+    def add_middleware(callable, args=[])
       @middleware << [callable, args]
     end
 
     def start
-      @server.listen(10)
-      loop do
-        client = @server.accept
+      busy_threads = 0
+      @server.listen(@tcp_backlog)
 
-        app_thread = Thread.new do
-          error = nil
-          begin
-            process_new_client(client)
-          rescue Exception => e
-            error = e
+      loop do
+        time_start = Process.clock_gettime(:CLOCK_MONOTONIC, :float_second)
+        loop do
+          current_busy = @tp_sync.synchronize { busy_threads }
+          break if current_busy < @max_threads
+          sleep(0)
+          current_time = Process.clock_gettime(:CLOCK_MONOTONIC, :float_second)
+          if (current_time - time_start) > 1.0
+            # puts "Waiting a long time: #{(current_time - time_start)}"
+            sleep 0.2
           end
+        end
 
-          if error
-            puts error.full_message
+        client = @server.accept
+        @tp_sync.synchronize { busy_threads += 1 }
 
-            response = Response.make_generic :internal_server_error
-            dispatch_response(client, response)
-          end
+        Thread.new do
+          begin
+            error, response = nil
+            begin
+              response = process_new_client(client)
+            rescue Exception => e
+              error = e
+            end
 
-          client.close rescue nil
-        end
+            if error
+              puts error.full_message
 
-        over_capacity = true
-        added_thread_to_list = false
-        while over_capacity
-          @tp_sync.synchronize do
-            unless added_thread_to_list
-              @threads << app_thread
-              added_thread_to_list = true
+              response = Response.make_generic :internal_server_error
+              dispatch_response(client, response)
             end
-            over_capacity = (@threads.length >= @max_threads)
-            # puts("#{@threads.length} of #{@max_threads}")
 
-            @threads.delete_if { |t| !t.alive? }
+            if response&.hijack
+              response.hijack.call(client)
+            else
+              client.close rescue nil
+            end
+          ensure
+            @tp_sync.synchronize { busy_threads -= 1 }
           end
-
-          # puts("XXX OVER CAPACITY!") if over_capacity
-          sleep 0 if over_capacity
         end
       end
     end
@@ -97,22 +103,30 @@ module Spyder
       content_length = response.headers.dict['content-length']
       if !content_length && response.body && response.body.is_a?(String)
         content_length = response.body.length
+        response.set_header 'content-length', content_length.to_s
       end
 
-      socket.write("HTTP/1.1 #{response.code} #{response.reason_sentence.b}\r\n")
-      response.headers.ordered.each do |name, value|
-        socket.write("#{name.b}: #{value.b}\r\n")
-      end
-      socket.write("connection: close\r\n") # FIXME:
-      socket.write("content-length: #{content_length}\r\n") if content_length
-      socket.write("\r\n")
-
-      if response.body
-        Array(response.body).each do |part|
-          content = part.respond_to?(:call) ? part.call : part
-          socket.write(content.b)
+      response.set_header('connection', 'close') unless response.headers.dict['connection']
+
+      begin
+        socket.write("HTTP/1.1 #{response.code} #{response.reason_sentence.b}\r\n")
+        response.headers.ordered.each do |name, value|
+          socket.write("#{name.b}: #{value.b}\r\n")
         end
+        socket.write("\r\n")
+
+        if response.body
+          Array(response.body).each do |part|
+            content = part.respond_to?(:call) ? part.call : part
+            socket.write(content.b)
+          end
+        end
+      rescue Errno::EPIPE
+        # socket closed. So what?
+        socket.close rescue nil
       end
+
+      response
     end
 
     def read_line(socket)

data/lib/spyder/version.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
 module Spyder
-  VERSION = '0.1.1'
+  VERSION = '0.2.0'
 end
-

data/lib/spyder/web/file_server.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Spyder
+  module Web
+    class FileServer
+      attr_reader :base_paths
+      attr_reader :default_index
+
+      def initialize(paths, index: nil)
+        @default_index = index
+        @base_paths = Array(paths).map do |path|
+          File.expand_path(File.join(Dir.pwd, path))
+        end
+      end
+
+      def call(request)
+        return unless request.verb == 'GET'
+
+        input_path = request.path
+        input_path = @default_index if request.path == '/' && @default_index
+
+        req_path = safe_request_path(input_path)
+        full_path = nil
+        return unless @base_paths.any? do |base|
+          fp = File.join(base, *req_path)
+          next unless File.file?(fp)
+          full_path = fp
+
+          true
+        end
+
+        st = File.readable?(full_path) ? File.stat(full_path) : nil
+
+        return unless st && %w[file link].include?(st.ftype)
+
+        etag = "\"#{"%xT-%x0" % [st.mtime, st.size]}\""
+
+        resp = serve_not_modified_response(request, st, etag)
+        return resp if resp
+
+        resp = Spyder::Response.new
+        resp.add_standard_headers
+        resp.set_header 'last-modified', st.mtime.httpdate
+        resp.set_header 'etag', etag
+        resp.set_header 'cache-control', 'public, must-revalidate, max-age=0'
+
+        File.open full_path do |fp|
+          mime = Marcel::MimeType.for(fp)
+          if mime == 'application/octet-stream' || mime == 'text/plain'
+            mime = Marcel::MimeType.for(name: req_path.last)
+          end
+
+          resp.set_header('content-type', mime) if mime
+
+          fp.rewind
+          resp.body = fp.read
+        end
+
+        resp
+      end
+
+      private
+
+      def serve_not_modified_response(request, st, etag)
+        if_etag = request.headers.dict['if-none-match']
+        if_modified_since = request.headers.dict['if-modified-since']
+
+        return unless if_etag || if_modified_since
+
+        resp = Spyder::Response.new
+        resp.add_standard_headers
+        resp.code = 304
+
+        return resp if if_etag && if_etag == etag
+
+        if_ms = if_modified_since ? Time.parse(if_modified_since) : nil
+
+        (if_ms && Time.at(st.mtime.to_i) <= if_ms) ? resp  : nil
+      end
+
+      def safe_request_path(path)
+        path.split('/').reject do |component|
+          component == '..' || component == '' || component == '.'
+        end
+      end
+    end
+  end
+end

data/lib/spyder/web_socket.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Spyder
+  class WebSocket
+    WS_CONST = '258EAFA5-E914-47DA-95CA-C5AB0DC85B11'
+
+    class Frame
+      def initialize
+      end
+
+      def decode(raw_data)
+        fin = raw_data[0] & 1
+        rsv = (raw_data[0] & 0b0111) >> 1
+      end
+    end
+
+    def initialize
+      @socket = nil
+      @on_start = nil
+      @on_message = nil
+      @on_close = nil
+
+      @streaming_buffer = WebSocketStreamingBuffer.new do |frame, mode, fragmented, last_fragment|
+        @on_message.call(frame, mode) unless fragmented
+      end
+      @streaming_buffer.on_close = proc do
+        @socket.close rescue nil
+      end
+    end
+
+    def send_text(data)
+      send_data(data, :text)
+    end
+
+    def send_binary(data)
+      send_data(data, :binary)
+    end
+
+    def send_data(data, mode)
+      data = data.b
+
+      length = data.length
+      buffer = String.new(encoding: 'ascii-8bit', capacity: length + 32)
+      buffer += ((1 << 7) | (mode == :binary ? 2 : 1)).chr
+      if length < 126
+        buffer += length.chr
+      elsif length <= 0xFFFF
+        buffer += [126, length].pack('CS>')
+      else
+        buffer += [127, length].pack('CQ>')
+      end
+
+      @socket.write(buffer)
+      @socket.write(data)
+    end
+
+    def on_close(&blk)
+      @on_close = blk
+    end
+
+    def on_message(&blk)
+      @on_message = blk
+    end
+
+    def on_start(&blk)
+      @on_start = blk
+    end
+
+    def hijacked!(socket)
+      @socket = socket
+      puts "websocket hijacked! #{socket}"
+      @thread = Thread.start do
+        self.threaded_start!
+      end
+    end
+
+    def self.upgrade_websocket_request(request)
+      ws_key = request.headers.dict['sec-websocket-key']
+      return unless ws_key && request.headers.dict['upgrade'] == 'websocket'
+
+      conns = request.headers.dict.fetch('connection', '').split(' ').map(&:strip)
+      return unless conns.include?('Upgrade')
+
+      ws_version = request.headers.dict['sec-websocket-version'] # expect: 13
+      return unless ws_version
+
+      protocols = request.headers.dict['sec-websocket-protocol']
+      protocols = protocols.split(',').map(&:strip) if protocols
+
+      extensions = request.headers.dict['sec-websocket-extensions']
+      extensions = extensions.split(';').map(&:strip) if extensions
+
+      decoded_key = Base64.strict_decode64(ws_key) rescue nil
+      return unless decoded_key&.length == 16
+
+      response_key = Base64.strict_encode64(
+        OpenSSL::Digest::SHA1.digest("#{ws_key}#{WS_CONST}")
+      )
+
+      ws = new
+
+      chosen_proto = yield(ws, protocols) if (block_given? && protocols)
+      chosen_proto = protocols.first if !chosen_proto && protocols
+
+      resp = Spyder::Response.new(code: 101)
+      resp.add_standard_headers
+      resp.set_header 'connection', 'Upgrade'
+      resp.set_header 'upgrade', 'websocket'
+      resp.set_header('sec-websocket-protocol', chosen_proto) if chosen_proto
+      resp.set_header 'sec-websocket-accept', response_key
+
+      resp.hijack = proc { |client_socket| ws.hijacked!(client_socket) }
+
+      [resp, ws]
+    end
+
+    def threaded_start!
+      @on_start.call if @on_start
+
+      while !@socket.closed? && !@socket.eof?
+        data = nil
+        begin
+          data = @socket.read_nonblock(16 * 1024)
+        rescue IO::WaitReadable
+          IO.select([@socket], [], [@socket])
+        end
+
+        next unless data
+
+        puts "ws: read #{data.length} bytes"
+        @streaming_buffer.feed(data)
+      end
+
+      @on_close.call unless @on_close
+    end
+  end
+end

data/lib/spyder/web_socket_streaming_buffer.rb

@@ -0,0 +1,192 @@
+# frozen_string_literal: true
+
+module Spyder
+  class WebSocketStreamingBuffer
+    attr_reader :buffer
+    attr_accessor :on_close
+
+    def initialize(&frame_callback)
+      @frame_callback = frame_callback
+      @on_close = nil
+      @buffer = []
+      _reset
+    end
+
+    def feed(buf)
+      @buffer += buf.bytes
+
+      # printable = buf.bytes.map{ |x| x.to_s(16).rjust(2, '0').upcase }.join(' ')
+      # puts "Got #{buf.bytes.length} more bytes: [#{printable}]"
+
+      iterations = 0
+      loop do
+        _flush
+
+        iterations += 1
+        if iterations > 1_000
+          raise "Watchdog error: got stuck on a loop of #{iterations} iterations."
+        end
+
+        break if (@buffer.length == 0 && !@need_buffer_length) ||
+          (@need_buffer_length && @buffer.length < @need_buffer_length)
+      end
+    end
+
+    private
+
+    def _reset
+      @state = :initial
+      @mask = nil
+      @opcode = nil
+      @data_mode = nil
+      @payload_length = nil
+      @need_buffer_length = nil
+      @flag_fin = nil
+      @fragmented = false
+      @fragmented_total_size = nil
+    end
+
+    def _flush
+      send(:"_flush_#{@state}")
+    end
+
+    def _flush_data
+      return unless @buffer.length >= @payload_length
+
+      data = if @mask
+        (0...@payload_length).map do |i|
+          @buffer[i] ^ @mask[i & 3]
+        end
+      else
+        @buffer[0...@payload_length]
+      end
+
+      last_fragment = @fragmented && @flag_fin
+      @frame_callback.call(data, @data_mode, @fragmented, last_fragment)
+
+      @buffer.shift(@payload_length)
+
+      if @fragmented && !last_fragment
+        @need_buffer_length = 2
+        @opcode = nil
+        @payload_length = nil
+        @state = "initial"
+      else
+        _reset
+      end
+    end
+
+    def close_with_error(msg)
+      @socket.close rescue nil
+
+      puts "Closing websocket: #{msg}"
+
+      @on_close.call if @on_close
+
+      _reset
+
+      @state = :closed
+      @socket = nil
+      @buffer = nil
+    end
+
+    def _flush_readhdr
+      return unless @buffer.length >= @need_buffer_length
+
+      @flag_fin = ((@buffer[0] & 0x80) != 0)
+      @opcode = (@buffer[0] & 0xF)
+      data_len = (@buffer[1] & 0x7F)
+      mask_flag = ((@buffer[1] & 0x80) != 0)
+
+      is_control_frame = ((@opcode & 8) != 0)
+      if is_control_frame && (data_len > 125 || !@flag_fin)
+        # Control frames must not be fragmented and must be <= 125 bytes in size
+        return close_with_error("Control frame invalid state")
+      end
+
+      if @fragmented && !is_control_frame && @opcode != 0
+        # We're in the middle of a fragmented frame and received a non-control
+        # frame. That's an error.
+        return close_with_error("Got non-control frame during fragmented frame.")
+      end
+
+      if !@fragmented && !@flag_fin && !is_control_frame
+        @fragmented = true
+        @fragmented_total_size = 0
+      end
+
+      shift_length = 2
+
+      @payload_length = if data_len == 127
+        shift_length += 8
+        @buffer[2...10].pack('C*').unpack('Q>').first
+      elsif data_len == 126
+        shift_length += 2
+        @buffer[2...4].pack('C*').unpack('S>').first
+      else
+        shift_length += 0
+        data_len
+      end
+
+      if mask_flag
+        @mask = @buffer[shift_length...(shift_length + 4)].pack('C*').bytes
+        shift_length += 4
+      else
+        @mask = nil
+      end
+
+      @fragmented_total_size += @payload_length if @fragmented
+
+      fgr = nil
+      fgr = ", fgr=#{@fragmented_total_size}" if @fragmented
+      puts "frame(code=#{@opcode}, len=#{@payload_length}, fin=#{@flag_fin}#{fgr})"
+
+      @buffer.shift(shift_length)
+      @need_buffer_length = ((0..2) === @opcode ? @payload_length : nil)
+
+      case @opcode
+      when 0x0 # continuation
+        @state = "data"
+      when 0x1 # text frame
+        @state = "data"
+        @data_mode = :text
+      when 0x2 # binary frame
+        @state = "data"
+        @data_mode = :binary
+      when (0x3..0x7) # reserved
+        unexpected_opcode(@opcode)
+      when 0x8 # connection close
+        @on_close.call if @on_close
+
+        _reset
+      when 0x9 # ping
+        # TODO: send pong?
+      when 0xA # pong
+        puts "WS: got pong!"
+
+        _reset
+      else
+        unexpected_opcode(@opcode)
+      end
+    end
+
+    def _flush_initial
+      return unless @buffer.length >= 2
+
+      data_len = (@buffer[1] & 0x7F)
+      mask_flag = (@buffer[1] & 0x80)
+
+      @need_buffer_length = 2 + (mask_flag ? 4 : 0) + (
+        data_len == 127 ? 8 : (data_len == 126 ? 2 : 0)
+      )
+
+      @state = "readhdr"
+      _flush
+    end
+
+    def unexpected_opcode opcode
+      puts "WS: unexpected opcode #{opcode}"
+      exit 1
+    end
+  end
+end

data/lib/spyder.rb

@@ -13,3 +13,12 @@ require 'spyder/request'
 require 'spyder/response'
 require 'spyder/router'
 require 'spyder/server'
+require 'spyder/web_socket_streaming_buffer'
+require 'spyder/web_socket'
+
+# spyder-web
+require 'marcel'
+require 'base64'
+require 'openssl'
+
+require 'spyder/web/file_server'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: spyder
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.2.0
 platform: ruby
 authors:
 - André D. Piske
-autorequire:
+autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-02-11 00:00:00.000000000 Z
+date: 2025-06-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: mustermann
@@ -24,6 +24,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '3.0'
+- !ruby/object:Gem::Dependency
+  name: marcel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.0'
 description: Spyder Web
 email: andrepiske@gmail.com
 executables: []
@@ -39,11 +53,14 @@ files:
 - lib/spyder/router.rb
 - lib/spyder/server.rb
 - lib/spyder/version.rb
+- lib/spyder/web/file_server.rb
+- lib/spyder/web_socket.rb
+- lib/spyder/web_socket_streaming_buffer.rb
 homepage: https://github.com/andrepiske/spyder
 licenses:
 - MIT
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -58,8 +75,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.6
-signing_key:
+rubygems_version: 3.0.3.1
+signing_key: 
 specification_version: 4
 summary: Spyder
 test_files: []