skyfall 0.1.3 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b1d8eea31c1c63f4aabd513a470083862b653f4fb1ba5a0a638da1d9e8cba3af
-  data.tar.gz: 15e57a872800ad3a0c2a0f38be1ed31694617e82fe0981e1c9a1598c9f70e325
+  metadata.gz: 9a508caa504715c30f9d9ff68dbf419c8b88ad00e5fbc6315ae6a3abb624e167
+  data.tar.gz: 9010ece05b7f1cce0a2573ecfa75965d1dba2b77cd7926eaa8fc510f8e2ae7e2
 SHA512:
-  metadata.gz: 84544c24ff44ed583df898567b260db3b23517de6248b105f6199de49f2a3bcfaaada701bb62252c59a44af2845321637cfa4ab5061b9f8f800d27107fb2f83e
-  data.tar.gz: fb62308d1e517fdd257633a19f503dc9f9fce7d5b6b4d778a588c19facbe8c45ff60cca3e242ae9024c4975c9ca84326083e4eeddedd0da8d34f3ae9f647d05b
+  metadata.gz: b9fc0370411496e76a4caaa7b2795e999b2fbf3445e21a02e3b9135e4c7fc5e3766d7ea3a94e145becf3ab96fd4d494b0f7d8a04a09eee03698a97125d44238d
+  data.tar.gz: b302a5f7eefc30509a6b721809105bd9439f6919b31ab73878d132fa8a67062d0d77c17d375d3d433cb77bbba63127a823d1817f5d1a71f16c2b6601dca7d493

data/CHANGELOG.md

@@ -1,3 +1,21 @@
+## [0.2.1] - 2023-08-19
+
+- optimized `WebsocketMessage` parsing performance - lazy parsing of most properties (message decoding should be over 50% faster on average)
+- added separate subclasses of `WebsocketMessage` for different message types
+- added support for `#handle`, `#info` and `#tombstone` message types
+- `UnknownMessage` is returned for unrecognized message types
+
+## [0.2.0] - 2023-07-24
+
+- switched the websocket library from `websocket-client-simple` to `faye-websocket`, which should make event parsing up to ~30× faster (!)
+- added `auto_reconnect` property to `Stream` (on by default) - if true, it will try to reconnect with an exponential backoff when the websocket disconnects, until you call `Stream#disconnect`
+
+Note:
+
+- calling `sleep` is no longer needed after connecting - call `connect` on a new thread instead to get previously default behavior of running the event loop asynchronously
+- the disconnect event no longer passes an error object in the argument
+- there is currently no "heartbeat" feature as in 0.1.x that checks for a stuck connection - but it doesn't seem to be needed
+
 ## [0.1.3] - 2023-07-04
 
 - allow passing a previously saved cursor to websocket to replay any missed events

data/README.md

@@ -41,8 +41,6 @@ When you're ready, open the connection by calling `connect`:
 sky.connect
 ```
 
-The connection is started asynchronously on a separate thread. If you're running this code in a simple script (and not as a part of a server), call e.g. `sleep` without arguments or `loop { STDIN.read }` to prevent the script for exiting while the connection is open.
-
 
 ### Processing messages
 

data/example/firehose.rb

@@ -21,7 +21,7 @@ end
 
 sky.on_connect { puts "Connected" }
 sky.on_disconnect { puts "Disconnected" }
+sky.on_reconnect { puts "Reconnecting..." }
 sky.on_error { |e| puts "ERROR: #{e}" }
 
 sky.connect
-sleep

data/lib/skyfall/car_archive.rb

@@ -31,6 +31,9 @@ module Skyfall
       read_section(buffer) until buffer.eof?
     end
 
+    def section_with_cid(cid)
+      @sections.detect { |s| s.cid == cid }&.body
+    end
 
     private
 

data/lib/skyfall/messages/commit_message.rb

@@ -0,0 +1,23 @@
+require_relative '../car_archive'
+require_relative '../cid'
+require_relative '../operation'
+
+module Skyfall
+  class CommitMessage < WebsocketMessage
+    def commit
+      @commit ||= @data_object['commit'] && CID.from_cbor_tag(@data_object['commit'])
+    end
+
+    def prev
+      @prev ||= @data_object['prev'] && CID.from_cbor_tag(@data_object['prev'])
+    end
+
+    def blocks
+      @blocks ||= CarArchive.new(@data_object['blocks'])
+    end
+
+    def operations
+      @operations ||= @data_object['ops'].map { |op| Operation.new(self, op) }
+    end
+  end
+end

data/lib/skyfall/messages/handle_message.rb

@@ -0,0 +1,7 @@
+module Skyfall
+  class HandleMessage < WebsocketMessage
+    def handle
+      @data_object['handle']
+    end
+  end
+end

data/lib/skyfall/messages/info_message.rb

@@ -0,0 +1,22 @@
+module Skyfall
+  class InfoMessage < WebsocketMessage
+    attr_reader :name, :message
+
+    OUTDATED_CURSOR = "OutdatedCursor"
+
+    def initialize(type_object, data_object)
+      super
+
+      @name = @data_object['name']
+      @message = @data_object['message']
+    end
+
+    def to_s
+      (@name || "InfoMessage") + (@message ? ": #{@message}" : "")
+    end
+
+    def inspectable_variables
+      super - [:@did, :@seq]
+    end
+  end
+end

data/lib/skyfall/messages/tombstone_message.rb

@@ -0,0 +1,4 @@
+module Skyfall
+  class TombstoneMessage < WebsocketMessage
+  end
+end

data/lib/skyfall/messages/unknown_message.rb

@@ -0,0 +1,4 @@
+module Skyfall
+  class UnknownMessage < WebsocketMessage
+  end
+end

data/lib/skyfall/messages/websocket_message.rb

@@ -0,0 +1,90 @@
+require_relative '../errors'
+require_relative '../extensions'
+
+require 'cbor'
+require 'time'
+
+module Skyfall
+  class WebsocketMessage
+    using Skyfall::Extensions
+
+    require_relative 'commit_message'
+    require_relative 'handle_message'
+    require_relative 'info_message'
+    require_relative 'tombstone_message'
+    require_relative 'unknown_message'
+
+    attr_reader :type_object, :data_object
+    attr_reader :type, :did, :seq
+
+    alias repo did
+
+    def self.new(data)
+      type_object, data_object = decode_cbor_objects(data)
+
+      message_class = case type_object['t']
+        when '#commit' then CommitMessage
+        when '#handle' then HandleMessage
+        when '#info' then InfoMessage
+        when '#tombstone' then TombstoneMessage
+        else UnknownMessage
+      end
+
+      message = message_class.allocate
+      message.send(:initialize, type_object, data_object)
+      message
+    end
+
+    def initialize(type_object, data_object)
+      @type_object = type_object
+      @data_object = data_object
+
+      @type = @type_object['t'][1..-1].to_sym
+      @did = @data_object['repo'] || @data_object['did']
+      @seq = @data_object['seq']
+    end
+
+    def operations
+      []
+    end
+
+    def time
+      @time ||= @data_object['time'] && Time.parse(@data_object['time'])
+    end
+
+    def inspectable_variables
+      instance_variables - [:@type_object, :@data_object, :@blocks]
+    end
+
+    def inspect
+      vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
+      "#<#{self.class}:0x#{object_id} #{vars}>"
+    end
+
+    private
+
+    def self.decode_cbor_objects(data)
+      objects = CBOR.decode_sequence(data)
+
+      if objects.length < 2
+        raise DecodeError.new("Malformed message: #{objects.inspect}")
+      elsif objects.length > 2
+        raise DecodeError.new("Invalid number of objects: #{objects.length}")
+      end
+
+      type, data = objects
+
+      if data['error']
+        raise SubscriptionError.new(data['error'], data['message'])
+      end
+
+      raise DecodeError.new("Invalid object type: #{type}") unless type.is_a?(Hash)
+      raise UnsupportedError.new("Unexpected CBOR object: #{type}") unless type['op'] == 1
+      raise DecodeError.new("Missing data: #{type} #{objects.inspect}") unless type['op'] && type['t']
+      raise DecodeError.new("Invalid message type: #{type['t']}") unless type['t'].start_with?('#')
+      raise DecodeError.new("Invalid object type: #{data}") unless data.is_a?(Hash)
+
+      [type, data]
+    end
+  end
+end

data/lib/skyfall/operation.rb

@@ -2,30 +2,41 @@ require_relative 'collection'
 
 module Skyfall
   class Operation
-    attr_reader :repo, :path, :action, :cid
+    def initialize(message, json)
+      @message = message
+      @json = json
+    end
 
-    def initialize(repo, path, action, cid, record)
-      @repo = repo
-      @path = path
-      @action = action.to_sym
-      @cid = cid
-      @record = record
+    def repo
+      @message.repo
     end
 
-    def raw_record
-      @record
+    def path
+      @json['path']
     end
 
-    def uri
-      "at://#{repo}/#{path}"
+    def action
+      @json['action'].to_sym
     end
 
     def collection
-      path.split('/')[0]
+      @json['path'].split('/')[0]
     end
 
     def rkey
-      path.split('/')[1]
+      @json['path'].split('/')[1]
+    end
+
+    def uri
+      "at://#{repo}/#{path}"
+    end
+
+    def cid
+      @cid ||= @json['cid'] && CID.from_cbor_tag(@json['cid'])
+    end
+
+    def raw_record
+      @raw_record ||= cid && @message.blocks.section_with_cid(cid)
     end
 
     def type

data/lib/skyfall/stream.rb

@@ -1,7 +1,8 @@
-require_relative 'websocket_message'
+require_relative 'messages/websocket_message'
 
+require 'eventmachine'
+require 'faye/websocket'
 require 'uri'
-require 'websocket-client-simple'
 
 module Skyfall
   class Stream
@@ -11,7 +12,9 @@ module Skyfall
       :subscribe_repos => SUBSCRIBE_REPOS
     }
 
-    attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor
+    MAX_RECONNECT_INTERVAL = 300
+
+    attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor, :auto_reconnect
 
     def initialize(server, endpoint, cursor = nil)
       @endpoint = check_endpoint(endpoint)
@@ -22,93 +25,71 @@ module Skyfall
       @heartbeat_interval = 5
       @heartbeat_timeout = 30
       @last_update = nil
+      @auto_reconnect = true
+      @connection_attempts = 0
     end
 
     def connect
-      return if @websocket
+      return if @ws
 
       url = build_websocket_url
-      handlers = @handlers
-      stream = self
-
-      handlers[:connecting]&.call(url)
-
-      @websocket = WebSocket::Client::Simple.connect(url) do |ws|
-        ws.on :message do |msg|
-          stream.notify_heartbeat
 
-          atp_message = Skyfall::WebsocketMessage.new(msg.data)
-          stream.cursor = atp_message.seq
+      @handlers[:connecting]&.call(url)
+      @engines_on = true
 
-          handlers[:raw_message]&.call(msg.data)
-          handlers[:message]&.call(atp_message)
-        end
-
-        ws.on :open do
-          handlers[:connect]&.call
+      EM.run do
+        EventMachine.error_handler do |e|
+          @handlers[:error]&.call(e)
         end
 
-        ws.on :close do |e|
-          handlers[:disconnect]&.call(e)
-        end
+        @ws = Faye::WebSocket::Client.new(url)
 
-        ws.on :error do |e|
-          handlers[:error]&.call(e)
+        @ws.on(:open) do |e|
+          @handlers[:connect]&.call
         end
-      end
 
-      if @heartbeat_interval && @heartbeat_timeout && @heartbeat_thread.nil?
-        hb_interval = @heartbeat_interval
-        hb_timeout = @heartbeat_timeout
+        @ws.on(:message) do |msg|
+          @connection_attempts = 0
 
-        @last_update = Time.now
+          data = msg.data.pack('C*')
+          @handlers[:raw_message]&.call(data)
 
-        @heartbeat_thread = Thread.new do
-          loop do
-            sleep(hb_interval)
-            @heartbeat_mutex.synchronize do
-              if Time.now - @last_update > hb_timeout
-                force_restart
-              end
-            end
+          if @handlers[:message]
+            atp_message = Skyfall::WebsocketMessage.new(data)
+            @cursor = atp_message.seq
+            @handlers[:message].call(atp_message)
+          else
+            @cursor = nil
           end
         end
-      end
-    end
 
-    def force_restart
-      @websocket.close
-      @websocket = nil
+        @ws.on(:error) do |e|
+          @handlers[:error]&.call(e)
+        end
 
-      timeout = 5
+        @ws.on(:close) do |e|
+          @ws = nil
 
-      loop do
-        begin
-          @handlers[:reconnect]&.call
-          connect
-          break
-        rescue Exception => e
-          @handlers[:error]&.call(e)
-          sleep(timeout)
-          timeout *= 2
+          if @auto_reconnect && @engines_on
+            EM.add_timer(reconnect_delay) do
+              @connection_attempts += 1
+              @handlers[:reconnect]&.call
+              connect
+            end
+          else
+            @engines_on = false
+            @handlers[:disconnect]&.call
+            EM.stop_event_loop unless @ws
+          end
         end
       end
-
-      @last_update = Time.now
     end
 
     def disconnect
-      return unless @websocket
-
-      @heartbeat_thread&.kill
-      @heartbeat_thread = nil
+      return unless EM.reactor_running?
 
-      @websocket.close
-      @websocket = nil
-    end
-
-    def notify_heartbeat
-      @heartbeat_mutex.synchronize { @last_update = Time.now }
+      @engines_on = false
+      EM.stop_event_loop
     end
 
     alias close disconnect
@@ -144,6 +125,14 @@ module Skyfall
 
     private
 
+    def reconnect_delay
+      if @connection_attempts == 0
+        0
+      else
+        [2 ** (@connection_attempts - 1), MAX_RECONNECT_INTERVAL].min
+      end
+    end
+
     def build_websocket_url
       url = "wss://#{@server}/xrpc/#{@endpoint}"
       url += "?cursor=#{@cursor}" if @cursor

data/lib/skyfall/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Skyfall
-  VERSION = "0.1.3"
+  VERSION = "0.2.1"
 end

data/lib/skyfall.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 require_relative 'skyfall/stream'
-require_relative 'skyfall/websocket_message'
+require_relative 'skyfall/messages/websocket_message'
 require_relative 'skyfall/version'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skyfall
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.1
 platform: ruby
 authors:
 - Kuba Suder
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-04 00:00:00.000000000 Z
+date: 2023-08-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base32
@@ -51,25 +51,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: 0.5.9.6
 - !ruby/object:Gem::Dependency
-  name: websocket-client-simple
+  name: faye-websocket
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.11.2
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.6'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.6.1
+        version: 0.11.2
 description: "\n    Skyfall is a Ruby library for connecting to the \"firehose\" of
   the Bluesky social network, i.e. a websocket which\n    streams all new posts and
   everything else happening on the Bluesky network in real time. The code connects
@@ -92,10 +86,15 @@ files:
 - lib/skyfall/collection.rb
 - lib/skyfall/errors.rb
 - lib/skyfall/extensions.rb
+- lib/skyfall/messages/commit_message.rb
+- lib/skyfall/messages/handle_message.rb
+- lib/skyfall/messages/info_message.rb
+- lib/skyfall/messages/tombstone_message.rb
+- lib/skyfall/messages/unknown_message.rb
+- lib/skyfall/messages/websocket_message.rb
 - lib/skyfall/operation.rb
 - lib/skyfall/stream.rb
 - lib/skyfall/version.rb
-- lib/skyfall/websocket_message.rb
 - sig/skyfall.rbs
 homepage: https://github.com/mackuba/skyfall
 licenses:

data/lib/skyfall/websocket_message.rb

@@ -1,77 +0,0 @@
-require_relative 'car_archive'
-require_relative 'cid'
-require_relative 'errors'
-require_relative 'extensions'
-require_relative 'operation'
-
-require 'cbor'
-require 'time'
-
-module Skyfall
-  class WebsocketMessage
-    using Skyfall::Extensions
-
-    attr_reader :type_object, :data_object
-    attr_reader :type, :repo, :time, :seq, :commit, :prev, :blocks, :operations
-
-    def initialize(data)
-      @type_object, @data_object = decode_cbor_objects(data)
-
-      @type = @type_object['t'][1..-1].to_sym
-      @operations = []
-
-      @repo = @data_object['repo']
-      @time = Time.parse(@data_object['time'])
-      @seq = @data_object['seq']
-
-      return unless @type == :commit
-
-      @commit = @data_object['commit'] && CID.from_cbor_tag(@data_object['commit'])
-      @prev = @data_object['prev'] && CID.from_cbor_tag(@data_object['prev'])
-
-      @blocks = CarArchive.new(@data_object['blocks'])
-
-      @operations = @data_object['ops'].map { |op|
-        cid = op['cid'] && CID.from_cbor_tag(op['cid'])
-        path = op['path']
-        action = op['action']
-        record = cid && @blocks.sections.detect { |s| s.cid == cid }.body
-
-        Operation.new(@repo, path, action, cid, record)
-      }
-    end
-
-    def inspect
-      keys = instance_variables - [:@type_object, :@data_object, :@blocks]
-      vars = keys.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
-      "#<#{self.class}:0x#{object_id} #{vars}>"
-    end
-
-    private
-
-    def decode_cbor_objects(data)
-      objects = CBOR.decode_sequence(data)
-
-      if objects.length < 2
-        raise DecodeError.new("Malformed message: #{objects.inspect}")
-      elsif objects.length > 2
-        raise DecodeError.new("Invalid number of objects: #{objects.length}")
-      end
-
-      type_object, data_object = objects
-
-      if data_object['error']
-        raise SubscriptionError.new(data_object['error'], data_object['message'])
-      end
-
-      raise DecodeError.new("Invalid object type: #{type_object}") unless type_object.is_a?(Hash)
-      raise UnsupportedError.new("Unexpected CBOR object: #{type_object}") unless type_object['op'] == 1
-      raise DecodeError.new("Missing data: #{type_object} #{objects.inspect}") unless type_object['op'] && type_object['t']
-      raise DecodeError.new("Invalid message type: #{type_object['t']}") unless type_object['t'].start_with?('#')
-
-      raise DecodeError.new("Invalid object type: #{data_object}") unless data_object.is_a?(Hash)
-
-      [type_object, data_object]
-    end
-  end
-end