skyfall 0.6.1 → 0.7.1

data/lib/skyfall/stream.rb

@@ -1,17 +1,85 @@
+# frozen_string_literal: true
+
 require 'eventmachine'
 require 'faye/websocket'
 require 'uri'
 
+require_relative 'errors'
+require_relative 'events'
 require_relative 'version'
 
 module Skyfall
+
+  # Base class of a websocket client. It provides basic websocket client functionality such as
+  # connecting to the service, keeping the connection alive and running lifecycle callbacks.
+  #
+  # In most cases, you will not create instances of this class directly, but rather use either
+  # {Firehose} or {Jetstream}. Use this class as a superclass if you need to implement some
+  # custom client for a websocket API that isn't supported yet.
+
   class Stream
-    EVENTS = %w(message raw_message connecting connect disconnect reconnect error timeout)
-    MAX_RECONNECT_INTERVAL = 300
+    extend Events
 
-    attr_accessor :auto_reconnect, :last_update, :user_agent
-    attr_accessor :heartbeat_timeout, :heartbeat_interval, :check_heartbeat
+    MAX_RECONNECT_INTERVAL = 300
 
+    # If enabled, the client will try to reconnect if the connection is closed unexpectedly.
+    # (Default: true)
+    #
+    # When the reconnect attempt fails, it will wait with an exponential backoff delay before
+    # retrying again, up to {MAX_RECONNECT_INTERVAL} seconds.
+    #
+    # @return [Boolean]
+    attr_accessor :auto_reconnect
+
+    # User agent sent in the header when connecting.
+    #
+    # Default value is {#default_user_agent} = {#version_string} `(Skyfall/x.y)`. It's recommended
+    # to set it or extend it with some information that indicates what service this is and who is
+    # running it (e.g. a Bluesky handle).
+    #
+    # @return [String]
+    # @example
+    #   client.user_agent = "my.service (@my.handle) #{client.version_string}"
+    attr_accessor :user_agent
+
+    # If enabled, runs a timer which does periodical "heatbeat checks".
+    #
+    # The heartbeat timer is started when the client connects to the service, and checks if the stream
+    # hasn't stalled and is still regularly sending new messages. If no messages are detected for some
+    # period of time, the client forces a reconnect.
+    #
+    # This is **not** enabled by default, because depending on the service you're connecting to, it
+    # might be normal to not receive any messages for a while.
+    #
+    # @see #heartbeat_timeout
+    # @see #heartbeat_interval
+    # @return [Boolean]
+    attr_accessor :check_heartbeat
+
+    # Interval in seconds between heartbeat checks (default: 10). Only used if {#check_heartbeat} is set.
+    # @return [Numeric]
+    attr_accessor :heartbeat_interval
+
+    # Number of seconds without messages after which reconnect is triggered (default: 300).
+    # Only used if {#check_heartbeat} is set.
+    # @return [Numeric]
+    attr_accessor :heartbeat_timeout
+
+    # Time when the most recent message was received from the websocket.
+    #
+    # Note: this is _local time_ when the message was received; this is different from the timestamp
+    # of the message, which is the server time of the original source (PDS) when emitting the message,
+    # and different from a potential `created_at` saved in the record.
+    #
+    # @return [Time, nil]
+    attr_reader :last_update
+
+    #
+    # @param server [String] Address of the server to connect to.
+    #   Expects a string with either just a hostname, or a ws:// or wss:// URL.
+    #
+    # @raise [ArgumentError] if the server parameter is invalid
+    #
     def initialize(server)
       @root_url = build_root_url(server)
 
@@ -27,9 +95,23 @@ module Skyfall
       @handlers[:error] = proc { |e| puts "ERROR: #{e}" }
     end
 
+    #
+    # Opens a connection to the configured websocket.
+    #
+    # This method starts an EventMachine reactor on the current thread, and will only return
+    # once the connection is closed.
+    #
+    # @return [nil]
+    # @raise [ConfigError] if no message handler has been configured
+    # @raise [ReactorActiveError] if another stream is already running
+    #
     def connect
       return if @ws
 
+      if @handlers[:message].nil? && @handlers[:raw_message].nil?
+        raise ConfigError, "Either on_message or on_raw_message handler needs to be set"
+      end
+
       url = build_websocket_url
 
       @handlers[:connecting]&.call(url)
@@ -86,11 +168,10 @@ module Skyfall
       end
     end
 
-    def handle_message(msg)
-      data = msg.data
-      @handlers[:raw_message]&.call(data)
-    end
-
+    #
+    # Forces a reconnect, closing the connection and calling {#connect} again.
+    # @return [nil]
+    #
     def reconnect
       @reconnecting = true
       @connection_attempts = 0
@@ -98,6 +179,10 @@ module Skyfall
       @ws ? @ws.close : connect
     end
 
+    #
+    # Closes the connection and stops the EventMachine reactor thread.
+    # @return [nil]
+    #
     def disconnect
       return unless EM.reactor_running?
 
@@ -108,10 +193,18 @@ module Skyfall
 
     alias close disconnect
 
+    #
+    # Default user agent sent when connecting to the service. (Currently `"#{version_string}"`)
+    # @return [String]
+    #
     def default_user_agent
       version_string
     end
 
+    #
+    # Skyfall version string for use in user agent strings (`"Skyfall/x.y"`).
+    # @return [String]
+    #
     def version_string
       "Skyfall/#{Skyfall::VERSION}"
     end
@@ -126,60 +219,172 @@ module Skyfall
       end
     end
 
-    def start_heartbeat_timer
-      return if !@check_heartbeat || @heartbeat_interval.to_f <= 0 || @heartbeat_timeout.to_f <= 0
-      return if @heartbeat_timer
 
-      @heartbeat_timer = EM::PeriodicTimer.new(@heartbeat_interval) do
-        next if @ws.nil? || @heartbeat_timeout.to_f <= 0
-        time_passed = Time.now - @last_update
-
-        if time_passed > @heartbeat_timeout
-          @handlers[:timeout]&.call
-          reconnect
-        end
-      end
+    # @!method on_connecting(block)
+    #   Defines a callback to be run when the client tries to open a connection to the websocket.
+    #   Can be also run as a setter `on_connecting=`.
+    #   @param [Proc] block
+    #   @yieldparam [String] url URL to which the client is connecting
+    #   @return [nil]
+
+    event_handler :connecting
+
+    # @!method on_connect(block)
+    #   Defines a callback to be run after a connection to the websocket is opened.
+    #   Can be also run as a setter `on_connect=`.
+    #   @param [Proc] block
+    #   @return [nil]
+
+    event_handler :connect
+
+    # @!method on_raw_message(block)
+    #   Defines a callback to be run when a message is received, passing a raw data packet as
+    #   received from the websocket (plain text or binary). Can be also run as a setter `on_raw_message=`.
+    #   @param [Proc] block
+    #   @yieldparam [String] data payload of the received message
+    #   @return [nil]
+
+    event_handler :raw_message
+
+    # @!method on_message(block)
+    #   Defines a callback to be run when a message is received, passing the message as a parsed
+    #   object of an appropriate message class. Can be also run as a setter `on_message=`.
+    #   @param [Proc] block
+    #   @yieldparam [Object] message parsed message of an appropriate class
+    #   @return [nil]
+
+    event_handler :message
+
+    # @!method on_disconnect(block)
+    #   Defines a callback to be run after a connection to the websocket is closed (and the client
+    #   does not reconnect). Can be also run as a setter `on_disconnect=`.
+    #
+    #   This callback is not run when `on_reconnect` fires.
+    #   @param [Proc] block
+    #   @return [nil]
+
+    event_handler :disconnect
+
+    # @!method on_reconnect(block)
+    #   Defines a callback to be run when a connection to the websocket is broken, but the client
+    #   initiates or schedules a reconnect (which may happen after a delay). Can be also run as
+    #   a setter `on_reconnect=`.
+    #   @param [Proc] block
+    #   @return [nil]
+
+    event_handler :reconnect
+
+    # @!method on_timeout(block)
+    #   Defines a callback to be run when the heartbeat timer forces a reconnect. A reconnect is
+    #   triggered after not receiving any messages for a period of time specified in {#heartbeat_timeout}
+    #   (if {#check_heartbeat} is enabled). Can be also run as a setter `on_timeout=`.
+    #
+    #   This callback is also followed by `on_reconnect`.
+    #   @param [Proc] block
+    #   @return [nil]
+
+    event_handler :timeout
+
+    # @!method on_error(block)
+    #   Defines a callback to be run when the websocket connection returns an error. Can be also
+    #   run as a setter `on_error=`.
+    #
+    #   Default handler prints the error to stdout.
+    #
+    #   @param [Proc] block
+    #   @yieldparam [Exception] error the received error
+    #   @return [nil]
+
+    event_handler :error
+
+
+    # Returns a string with a representation of the object for debugging purposes.
+    # @return [String]
+    def inspect
+      vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
+      "#<#{self.class}:0x#{object_id} #{vars}>"
     end
 
-    def stop_heartbeat_timer
-      @heartbeat_timer&.cancel
-      @heartbeat_timer = nil
-    end
 
-    EVENTS.each do |event|
-      define_method "on_#{event}" do |&block|
-        @handlers[event.to_sym] = block
-      end
+    protected
 
-      define_method "on_#{event}=" do |block|
-        @handlers[event.to_sym] = block
-      end
-    end
+    # @note This method is designed to be overridden in subclasses.
+    #
+    # Returns the full URL of the websocket endpoint to connect to, with path and query parameters
+    # if needed. The base implementation simply returns the base URL passed to the initializer.
+    #
+    # Override this method in subclasses to point to the specific endpoint and add necessary
+    # parameters like cursor or filters, depending on the arguments passed to the constructor.
+    #
+    # @return [String]
 
-    def inspectable_variables
-      instance_variables - [:@handlers, :@ws]
+    def build_websocket_url
+      @root_url
     end
 
-    def inspect
-      vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
-      "#<#{self.class}:0x#{object_id} #{vars}>"
+    # Builds and configures a websocket client object that is used to connect to the requested service.
+    #
+    # @return [Faye::WebSocket::Client]
+    #   see {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/Client Faye::WebSocket::Client}
+
+    def build_websocket_client(url)
+      Faye::WebSocket::Client.new(url, nil, { headers: { 'User-Agent' => user_agent }.merge(request_headers) })
     end
 
+    # @note This method is designed to be overridden in subclasses.
+    #
+    # Processes a single message received from the websocket. The implementation is expected to
+    # parse the message from a plain text or binary form, build an appropriate message object,
+    # and call the `:message` and/or `:raw_message` callback handlers, passing the right parameters.
+    #
+    # The base implementation simply takes the message data and passes it as is to `:raw_message`,
+    # and does not call `:message` at all.
+    #
+    # @param msg
+    #   {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/API/MessageEvent Faye::WebSocket::API::MessageEvent}
+    # @return [nil]
 
-    protected
+    def handle_message(msg)
+      data = msg.data
+      @handlers[:raw_message]&.call(data)
+    end
+
+    # Additional headers to pass with the request when connecting to the websocket endpoint.
+    # The user agent header (built from {#user_agent}) is added separately.
+    #
+    # The base implementation returns an empty hash.
+    #
+    # @return [Hash] a hash of `{ header_name => header_value }`
 
     def request_headers
       {}
     end
 
+    # Returns the underlying websocket client object. It can be used e.g. to send messages back
+    # to the server (but see also: {#send_data}).
+    #
+    # @return [Faye::WebSocket::Client]
+    #   see {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/Client Faye::WebSocket::Client}
+
     def socket
       @ws
     end
 
+    # Sends a message back to the server.
+    #
+    # @param data [String, Array] the message to send -
+    #   a string for text websockets, a binary string or byte array for binary websockets
+    # @return [Boolean] true if the message was sent successfully
+
     def send_data(data)
       @ws.send(data)
     end
 
+    # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
+    def inspectable_variables
+      instance_variables - [:@handlers, :@ws]
+    end
+
 
     private
 
@@ -187,6 +392,26 @@ module Skyfall
       EM.reactor_running? && !@engines_on
     end
 
+    def start_heartbeat_timer
+      return if !@check_heartbeat || @heartbeat_interval.to_f <= 0 || @heartbeat_timeout.to_f <= 0
+      return if @heartbeat_timer
+
+      @heartbeat_timer = EM::PeriodicTimer.new(@heartbeat_interval) do
+        next if @ws.nil? || @heartbeat_timeout.to_f <= 0
+        time_passed = Time.now - @last_update
+
+        if time_passed > @heartbeat_timeout
+          @handlers[:timeout]&.call
+          reconnect
+        end
+      end
+    end
+
+    def stop_heartbeat_timer
+      @heartbeat_timer&.cancel
+      @heartbeat_timer = nil
+    end
+
     def reconnect_delay
       if @connection_attempts == 0
         0
@@ -195,14 +420,6 @@ module Skyfall
       end
     end
 
-    def build_websocket_client(url)
-      Faye::WebSocket::Client.new(url, nil, { headers: { 'User-Agent' => user_agent }.merge(request_headers) })
-    end
-
-    def build_websocket_url
-      @root_url
-    end
-
     def build_root_url(server)
       if !server.is_a?(String)
         raise ArgumentError, "Server parameter should be a string"

data/lib/skyfall/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Skyfall
-  VERSION = "0.6.1"
+  VERSION = "0.7.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: skyfall
 version: !ruby/object:Gem::Version
-  version: 0.6.1
+  version: 0.7.1
 platform: ruby
 authors:
 - Kuba Suder
@@ -10,99 +10,60 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: base32
+  name: eventmachine
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
+        version: 1.2.7
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.3'
+        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 0.3.4
-- !ruby/object:Gem::Dependency
-  name: base64
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.1'
+        version: 1.2.7
 - !ruby/object:Gem::Dependency
-  name: cbor
+  name: faye-websocket
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.5.9.6
+        version: '0.12'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '0.5'
-    - - ">="
-      - !ruby/object:Gem::Version
-        version: 0.5.9.6
+        version: '0.12'
 - !ruby/object:Gem::Dependency
-  name: eventmachine
+  name: oxygene
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.7
+        version: 0.0.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.2'
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.2.7
-- !ruby/object:Gem::Dependency
-  name: faye-websocket
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.12'
-  type: :runtime
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '0.12'
-description: "\n    Skyfall is a Ruby library for connecting to the \"firehose\" of
-  the Bluesky social network, i.e. a websocket which\n    streams all new posts and
-  everything else happening on the Bluesky network in real time. The code connects
-  to the\n    websocket endpoint, decodes the messages which are encoded in some binary
-  formats, and returns the data as Ruby\n    objects, which you can filter and save
-  to some kind of database (e.g. in order to create a custom feed).\n  "
+        version: 0.0.1
+description: "\n    Skyfall is a Ruby gem for connecting to the \"firehose\" of the
+  AT Protocol network that Bluesky is built on,\n    i.e. a websocket which streams
+  all new posts and everything else happening on the network in real time. The code\n
+  \   connects to the websocket endpoint, decodes the messages which are encoded in
+  some binary formats, and returns\n    the data as Ruby objects and hashes, which
+  you can filter and save to some kind of database (e.g. in order\n    to create a
+  custom feed).\n  "
 email:
 - jakub.suder@gmail.com
 executables: []
@@ -113,22 +74,18 @@ files:
 - LICENSE.txt
 - README.md
 - lib/skyfall.rb
-- lib/skyfall/car_archive.rb
-- lib/skyfall/cid.rb
 - lib/skyfall/collection.rb
 - lib/skyfall/errors.rb
-- lib/skyfall/extensions.rb
+- lib/skyfall/events.rb
 - lib/skyfall/firehose.rb
 - lib/skyfall/firehose/account_message.rb
 - lib/skyfall/firehose/commit_message.rb
-- lib/skyfall/firehose/handle_message.rb
 - lib/skyfall/firehose/identity_message.rb
 - lib/skyfall/firehose/info_message.rb
 - lib/skyfall/firehose/labels_message.rb
 - lib/skyfall/firehose/message.rb
 - lib/skyfall/firehose/operation.rb
 - lib/skyfall/firehose/sync_message.rb
-- lib/skyfall/firehose/tombstone_message.rb
 - lib/skyfall/firehose/unknown_message.rb
 - lib/skyfall/jetstream.rb
 - lib/skyfall/jetstream/account_message.rb
@@ -162,7 +119,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.3
+rubygems_version: 3.6.9
 specification_version: 4
 summary: A Ruby gem for streaming data from the Bluesky/ATProto firehose
 test_files: []

data/lib/skyfall/car_archive.rb

@@ -1,122 +0,0 @@
-require_relative 'cid'
-require_relative 'errors'
-require_relative 'extensions'
-
-require 'base64'
-require 'cbor'
-require 'stringio'
-
-# CAR v1: https://ipld.io/specs/transport/car/carv1/
-# multicodec codes: https://github.com/multiformats/multicodec/blob/master/table.csv
-
-module Skyfall
-  class CarSection
-    attr_reader :cid
-
-    def initialize(cid, body_data)
-      @cid = cid
-      @body_data = body_data
-    end
-
-    def body
-      @body ||= CarArchive.convert_data(CBOR.decode(@body_data))
-    end
-  end
-
-  class CarArchive
-    using Skyfall::Extensions
-
-    attr_reader :roots, :sections
-
-    def initialize(data)
-      @sections = []
-
-      buffer = StringIO.new(data)
-      read_header(buffer)
-      read_section(buffer) until buffer.eof?
-    end
-
-    def section_with_cid(cid)
-      section = @sections.detect { |s| s.cid == cid }
-      section && section.body
-    end
-
-    def self.convert_data(object)
-      if object.is_a?(Hash)
-        object.each do |k, v|
-          if v.is_a?(Hash) || v.is_a?(Array)
-            convert_data(v)
-          elsif v.is_a?(CBOR::Tagged)
-            object[k] = make_cid_link(v)
-          elsif v.is_a?(String) && v.encoding == Encoding::ASCII_8BIT
-            object[k] = make_bytes(v)
-          end
-        end
-      elsif object.is_a?(Array)
-        object.each_with_index do |v, i|
-          if v.is_a?(Hash) || v.is_a?(Array)
-            convert_data(v)
-          elsif v.is_a?(CBOR::Tagged)
-            object[i] = make_cid_link(v)
-          elsif v.is_a?(String) && v.encoding == Encoding::ASCII_8BIT
-            object[i] = make_bytes(v)
-          end
-        end
-      else
-        raise DecodeError, "Unexpected value type in record: #{object}"
-      end
-    end
-
-    def self.make_cid_link(cid)
-      { '$link' => CID.from_cbor_tag(cid) }
-    end
-
-    def self.make_bytes(data)
-      { '$bytes' => Base64.encode64(data).chomp.gsub(/=+$/, '') }
-    end
-
-    private
-
-    def read_header(buffer)
-      len = buffer.read_varint
-
-      header_data = buffer.read(len)
-      raise DecodeError.new("Header too short: #{header_data}") unless header_data.length == len
-
-      header = CBOR.decode(header_data)
-      raise UnsupportedError.new("Unexpected CAR version: #{header['version']}") unless header['version'] == 1
-      @roots = header['roots'].map { |x| CID.from_cbor_tag(x) }
-    end
-
-    def read_section(buffer)
-      len = buffer.read_varint
-
-      section_data = buffer.read(len)
-      raise DecodeError.new("Section too short: #{section_data}") unless section_data.length == len
-
-      sbuffer = StringIO.new(section_data)
-
-      version = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID version: #{version}") unless version == 1
-
-      codec = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID codec: #{codec}") unless codec == 0x71  # dag-cbor
-
-      hash = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID hash: #{hash}") unless hash == 0x12  # sha2-256
-
-      clen = sbuffer.read_varint
-      raise UnsupportedError.new("Unexpected CID length: #{clen}") unless clen == 32
-
-      prefix = section_data[0...sbuffer.pos]
-
-      cid_data = sbuffer.read(clen)
-      raise DecodeError.new("CID too short: #{cid_data}") unless cid_data.length == clen
-
-      cid = CID.new(prefix + cid_data)
-
-      body_data = sbuffer.read
-      @sections << CarSection.new(cid, body_data)
-    end
-  end
-end