skyfall 0.1.2 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6bc7ba74da06a265f8b1af42633b4c462f87fbf5764b38505628d9255bd97831
-  data.tar.gz: 9bbb8e00563fdab6ee906ce82a3ddcde4936b8f0ee187da4d4f8032f5f344446
+  metadata.gz: ce06b8a7ef3783cd255621d6868a737735b782999a8cab76b70cf1f869e2cd1d
+  data.tar.gz: 9edea42903a17cb83ee717ea92c81e9a853b530a24f0bc803f70835b3bed1ba2
 SHA512:
-  metadata.gz: 440c94077709828c54814ed8a4d520dbb513755412fa5baaee443d55f2493833e5c9196ffbcee256a118b8bdc6e41d92479fa041514ab8d9d16978aa4ec1d4c7
-  data.tar.gz: b43da862f141b6ed476036eee85c8514a3b93a7a93bcaa16e089ee6149137c6b548639351445b2d3bddb3e2c458e8dc6e2c3eaa3f71d2efb9d92dad734f2fbf1
+  metadata.gz: 1b04bb91b734fe84d984af3c6a8c78959230d5ef865109882d86c5f0b2537f0a8b25678dcace7f5d8c26e560f871edba4de07f38273e2e7cf17088fd125a08e2
+  data.tar.gz: 5dffcf8a308789995d16293a9b8caba848374ab80c147a0ef87f440cb2c98887b4d81f3eb447d74c2bfc608ae448b54d9ada90556eb6e55cf59cc1df37e15390

data/CHANGELOG.md

@@ -1,3 +1,22 @@
+## [0.2.0] - 2023-07-24
+
+- switched the websocket library from `websocket-client-simple` to `faye-websocket`, which should make event parsing up to ~30× faster (!)
+- added `auto_reconnect` property to `Stream` (on by default) - if true, it will try to reconnect with an exponential backoff when the websocket disconnects, until you call `Stream#disconnect`
+
+Note:
+
+- calling `sleep` is no longer needed after connecting - call `connect` on a new thread instead to get previously default behavior of running the event loop asynchronously
+- the disconnect event no longer passes an error object in the argument
+- there is currently no "heartbeat" feature as in 0.1.x that checks for a stuck connection - but it doesn't seem to be needed
+
+## [0.1.3] - 2023-07-04
+
+- allow passing a previously saved cursor to websocket to replay any missed events
+- the cursor is also kept in memory and automatically used when reconnecting
+- added "connecting" callback with url as argument
+- fixed connecting to websocket when endpoint is given as a string
+- improved error handling during parsing
+
 ## [0.1.2] - 2023-06-15
 
 - added rkey property for Operation

data/README.md

@@ -41,8 +41,6 @@ When you're ready, open the connection by calling `connect`:
 sky.connect
 ```
 
-The connection is started asynchronously on a separate thread. If you're running this code in a simple script (and not as a part of a server), call e.g. `sleep` without arguments or `loop { STDIN.read }` to prevent the script for exiting while the connection is open.
-
 
 ### Processing messages
 

data/example/firehose.rb

@@ -21,7 +21,7 @@ end
 
 sky.on_connect { puts "Connected" }
 sky.on_disconnect { puts "Disconnected" }
+sky.on_reconnect { puts "Reconnecting..." }
 sky.on_error { |e| puts "ERROR: #{e}" }
 
 sky.connect
-sleep

data/lib/skyfall/errors.rb

@@ -4,4 +4,15 @@ module Skyfall
 
   class UnsupportedError < StandardError
   end
+
+  class SubscriptionError < StandardError
+    attr_reader :error_type, :error_message
+
+    def initialize(error_type, error_message = nil)
+      @error_type = error_type
+      @error_message = error_message
+
+      super("Subscription error: #{error_type}" + (error_message ? " (#{error_message})" : ""))
+    end
+  end
 end

data/lib/skyfall/stream.rb

@@ -1,5 +1,8 @@
 require_relative 'websocket_message'
-require 'websocket-client-simple'
+
+require 'eventmachine'
+require 'faye/websocket'
+require 'uri'
 
 module Skyfall
   class Stream
@@ -9,101 +12,84 @@ module Skyfall
       :subscribe_repos => SUBSCRIBE_REPOS
     }
 
-    attr_accessor :heartbeat_timeout, :heartbeat_interval
+    MAX_RECONNECT_INTERVAL = 300
+
+    attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor, :auto_reconnect
 
-    def initialize(server, endpoint)
+    def initialize(server, endpoint, cursor = nil)
       @endpoint = check_endpoint(endpoint)
       @server = check_hostname(server)
+      @cursor = cursor
       @handlers = {}
       @heartbeat_mutex = Mutex.new
       @heartbeat_interval = 5
       @heartbeat_timeout = 30
       @last_update = nil
+      @auto_reconnect = true
+      @connection_attempts = 0
     end
 
     def connect
-      return if @websocket
-
-      url = "wss://#{@server}/xrpc/#{@endpoint}"
-      handlers = @handlers
-      stream = self
+      return if @ws
 
-      @websocket = WebSocket::Client::Simple.connect(url) do |ws|
-        ws.on :message do |msg|
-          stream.notify_heartbeat
-          handlers[:raw_message]&.call(msg.data)
+      url = build_websocket_url
 
-          if handlers[:message]
-            atp_message = Skyfall::WebsocketMessage.new(msg.data)
-            handlers[:message].call(atp_message)
-          end
-        end
+      @handlers[:connecting]&.call(url)
+      @engines_on = true
 
-        ws.on :open do
-          handlers[:connect]&.call
+      EM.run do
+        EventMachine.error_handler do |e|
+          @handlers[:error]&.call(e)
         end
 
-        ws.on :close do |e|
-          handlers[:disconnect]&.call(e)
-        end
+        @ws = Faye::WebSocket::Client.new(url)
 
-        ws.on :error do |e|
-          handlers[:error]&.call(e)
+        @ws.on(:open) do |e|
+          @handlers[:connect]&.call
         end
-      end
 
-      if @heartbeat_interval && @heartbeat_timeout && @heartbeat_thread.nil?
-        hb_interval = @heartbeat_interval
-        hb_timeout = @heartbeat_timeout
+        @ws.on(:message) do |msg|
+          @connection_attempts = 0
 
-        @last_update = Time.now
+          data = msg.data.pack('C*')
+          @handlers[:raw_message]&.call(data)
 
-        @heartbeat_thread = Thread.new do
-          loop do
-            sleep(hb_interval)
-            @heartbeat_mutex.synchronize do
-              if Time.now - @last_update > hb_timeout
-                force_restart
-              end
-            end
+          if @handlers[:message]
+            atp_message = Skyfall::WebsocketMessage.new(data)
+            @cursor = atp_message.seq
+            @handlers[:message].call(atp_message)
+          else
+            @cursor = nil
           end
         end
-      end
-    end
 
-    def force_restart
-      @websocket.close
-      @websocket = nil
+        @ws.on(:error) do |e|
+          @handlers[:error]&.call(e)
+        end
 
-      timeout = 5
+        @ws.on(:close) do |e|
+          @ws = nil
 
-      loop do
-        begin
-          @handlers[:reconnect]&.call
-          connect
-          break
-        rescue Exception => e
-          @handlers[:error]&.call(e)
-          sleep(timeout)
-          timeout *= 2
+          if @auto_reconnect && @engines_on
+            EM.add_timer(reconnect_delay) do
+              @connection_attempts += 1
+              @handlers[:reconnect]&.call
+              connect
+            end
+          else
+            @engines_on = false
+            @handlers[:disconnect]&.call
+            EM.stop_event_loop unless @ws
+          end
         end
       end
-
-      @last_update = Time.now
     end
 
     def disconnect
-      return unless @websocket
-
-      @heartbeat_thread&.kill
-      @heartbeat_thread = nil
-
-      @websocket.close
-      @websocket = nil
-    end
+      return unless EM.reactor_running?
 
-    def notify_heartbeat
-      @heartbeat_mutex.synchronize { @last_update = Time.now }
+      @engines_on = false
+      EM.stop_event_loop
     end
 
     alias close disconnect
@@ -116,6 +102,10 @@ module Skyfall
       @handlers[:raw_message] = block
     end
 
+    def on_connecting(&block)
+      @handlers[:connecting] = block
+    end
+
     def on_connect(&block)
       @handlers[:connect] = block
     end
@@ -135,6 +125,20 @@ module Skyfall
 
     private
 
+    def reconnect_delay
+      if @connection_attempts == 0
+        0
+      else
+        [2 ** (@connection_attempts - 1), MAX_RECONNECT_INTERVAL].min
+      end
+    end
+
+    def build_websocket_url
+      url = "wss://#{@server}/xrpc/#{@endpoint}"
+      url += "?cursor=#{@cursor}" if @cursor
+      url
+    end
+
     def check_endpoint(endpoint)
       if endpoint.is_a?(String)
         raise ArgumentError("Invalid endpoint name: #{endpoint}") if endpoint.strip.empty? || !endpoint.include?('.')
@@ -144,6 +148,8 @@ module Skyfall
       else
         raise ArgumentError("Endpoint should be a string or a symbol")
       end
+
+      endpoint
     end
 
     def check_hostname(server)

data/lib/skyfall/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Skyfall
-  VERSION = "0.1.2"
+  VERSION = "0.2.0"
 end

data/lib/skyfall/websocket_message.rb

@@ -15,15 +15,7 @@ module Skyfall
     attr_reader :type, :repo, :time, :seq, :commit, :prev, :blocks, :operations
 
     def initialize(data)
-      objects = CBOR.decode_sequence(data)
-      raise DecodeError.new("Invalid number of objects: #{objects.length}") unless objects.length == 2
-
-      @type_object, @data_object = objects
-      raise DecodeError.new("Invalid object type: #{@type_object}") unless @type_object.is_a?(Hash)
-      raise DecodeError.new("Invalid object type: #{@data_object}") unless @data_object.is_a?(Hash)
-      raise DecodeError.new("Missing data: #{@type_object}") unless @type_object['op'] && @type_object['t']
-      raise DecodeError.new("Invalid message type: #{@type_object['t']}") unless @type_object['t'].start_with?('#')
-      raise UnsupportedError.new("Unexpected CBOR object: #{@type_object}") unless @type_object['op'] == 1
+      @type_object, @data_object = decode_cbor_objects(data)
 
       @type = @type_object['t'][1..-1].to_sym
       @operations = []
@@ -54,5 +46,32 @@ module Skyfall
       vars = keys.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
       "#<#{self.class}:0x#{object_id} #{vars}>"
     end
+
+    private
+
+    def decode_cbor_objects(data)
+      objects = CBOR.decode_sequence(data)
+
+      if objects.length < 2
+        raise DecodeError.new("Malformed message: #{objects.inspect}")
+      elsif objects.length > 2
+        raise DecodeError.new("Invalid number of objects: #{objects.length}")
+      end
+
+      type_object, data_object = objects
+
+      if data_object['error']
+        raise SubscriptionError.new(data_object['error'], data_object['message'])
+      end
+
+      raise DecodeError.new("Invalid object type: #{type_object}") unless type_object.is_a?(Hash)
+      raise UnsupportedError.new("Unexpected CBOR object: #{type_object}") unless type_object['op'] == 1
+      raise DecodeError.new("Missing data: #{type_object} #{objects.inspect}") unless type_object['op'] && type_object['t']
+      raise DecodeError.new("Invalid message type: #{type_object['t']}") unless type_object['t'].start_with?('#')
+
+      raise DecodeError.new("Invalid object type: #{data_object}") unless data_object.is_a?(Hash)
+
+      [type_object, data_object]
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skyfall
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.2.0
 platform: ruby
 authors:
 - Kuba Suder
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-06-15 00:00:00.000000000 Z
+date: 2023-07-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base32
@@ -51,25 +51,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.5.9.6
 - !ruby/object:Gem::Dependency
-  name: websocket-client-simple
+  name: faye-websocket
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.11.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.11.2
 description: "\n    Skyfall is a Ruby library for connecting to the \"firehose\" of
   the Bluesky social network, i.e. a websocket which\n    streams all new posts and
   everything else happening on the Bluesky network in real time. The code connects