skyfall 0.6.1 → 0.7.0

data/lib/skyfall/firehose/message.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require_relative '../errors'
 require_relative '../extensions'
 require_relative '../firehose'
@@ -6,45 +8,88 @@ require 'cbor'
 require 'time'
 
 module Skyfall
+
+  # @abstract
+  # Abstract base class representing a CBOR firehose message.
+  #
+  # Actual messages are returned as instances of one of the subclasses of this class,
+  # depending on the type of message, most commonly as {Skyfall::Firehose::CommitMessage}.
+  #
+  # The {new} method is overridden here so that it can be called with a binary data message
+  # from the websocket, and it parses the type from the appropriate frame and builds an
+  # instance of a matching subclass.
+  # 
+  # You normally don't need to call this class directly, unless you're building a custom
+  # subclass of {Skyfall::Stream}, or reading raw data packets from the websocket through
+  # the {Skyfall::Stream#on_raw_message} event handler.
+
   class Firehose::Message
     using Skyfall::Extensions
 
-    require_relative 'account_message'
-    require_relative 'commit_message'
-    require_relative 'handle_message'
-    require_relative 'identity_message'
-    require_relative 'info_message'
-    require_relative 'labels_message'
-    require_relative 'sync_message'
-    require_relative 'tombstone_message'
-    require_relative 'unknown_message'
-
-    attr_reader :type, :did, :seq
-    alias repo did
+    # Type of the message (e.g. `:commit`, `:identity` etc.)
+    # @return [Symbol]
+    attr_reader :type
 
-    # :nodoc: - consider this as semi-private API
-    attr_reader :type_object, :data_object
+    # DID of the account (repo) that the event is sent by.
+    # @return [String, nil]
+    attr_reader :did
 
+    # Sequential number of the message, to be used as a cursor when reconnecting.
+    # @return [Integer, nil]
+    attr_reader :seq
+
+    alias repo did
+    alias kind type
+
+    # First of the two CBOR objects forming the message payload, which mostly just includes the type field.
+    # @api private
+    # @return [Hash]
+    attr_reader :type_object
+
+    # Second of the two CBOR objects forming the message payload, which contains the rest of the data.
+    # @api private
+    # @return [Hash]
+    attr_reader :data_object
+
+    #
+    # Parses the CBOR objects from the binary data and returns an instance of an appropriate subclass.
+    # 
+    # {Skyfall::Firehose::UnknownMessage} is returned if the message type is not recognized.
+    #
+    # @param data [String] binary payload of a firehose websocket message
+    # @return [Skyfall::Firehose::Message]
+    # @raise [Skyfall::DecodeError] if the structure of the message is invalid
+    # @raise [Skyfall::UnsupportedError] if the message has an unknown future version
+    # @raise [Skyfall::SubscriptionError] if the data contains an error message from the server
+    #
     def self.new(data)
       type_object, data_object = decode_cbor_objects(data)
 
       message_class = case type_object['t']
         when '#account'   then Firehose::AccountMessage
         when '#commit'    then Firehose::CommitMessage
-        when '#handle'    then Firehose::HandleMessage
         when '#identity'  then Firehose::IdentityMessage
         when '#info'      then Firehose::InfoMessage
         when '#labels'    then Firehose::LabelsMessage
         when '#sync'      then Firehose::SyncMessage
-        when '#tombstone' then Firehose::TombstoneMessage
         else Firehose::UnknownMessage
       end
 
+      if self != Firehose::Message && self != message_class
+        expected_type = self.name.split('::').last.gsub(/Message$/, '').downcase
+        raise DecodeError, "Expected ##{expected_type} message, got #{type_object['t']}"
+      end
+
       message = message_class.allocate
       message.send(:initialize, type_object, data_object)
       message
     end
 
+    #
+    # @private
+    # @param type_object [Hash] first decoded CBOR frame with metadata
+    # @param data_object [Hash] second decoded CBOR frame with payload
+    #
     def initialize(type_object, data_object)
       @type_object = type_object
       @data_object = data_object
@@ -54,29 +99,79 @@ module Skyfall
       @seq = @data_object['seq']
     end
 
+    #
+    # List of operations on records included in the message. Only `#commit` messages include
+    # operations, but for convenience the method is declared here and returns an empty array
+    # in other messages.
+    # @return [Array<Firehose::Operation>]
+    #
     def operations
       []
     end
 
+    #
+    # @return [Boolean] true if the message is {Firehose::UnknownMessage} (of unrecognized type)
+    #
     def unknown?
       self.is_a?(Firehose::UnknownMessage)
     end
 
+    #
+    # Timestamp decoded from the message.
+    #
+    # Note: this represents the time when the message was emitted from the original PDS, which
+    # might differ a lot from the `created_at` time saved in the record data, e.g. if user's local
+    # time is set incorrectly, or if an archive of existing posts was imported from another platform.
+    #
+    # @return [Time, nil]
+    #
     def time
-      @time ||= @data_object['time'] && Time.parse(@data_object['time'])
+      @time ||= @data_object['time'] && Time.iso8601(@data_object['time'])
     end
 
-    def inspectable_variables
-      instance_variables - [:@type_object, :@data_object, :@blocks]
+    # Much faster version for Ruby 3.2+
+
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('3.2')
+      def time
+        @time ||= @data_object['time'] && Time.new(@data_object['time'])
+      end
     end
 
+    # Returns a string with a representation of the object for debugging purposes.
+    # @return [String]
     def inspect
       vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
       "#<#{self.class}:0x#{object_id} #{vars}>"
     end
 
+
+    protected
+
+    # @return [Array<Symbol>] list of instance variables to be printed in the {#inspect} output
+    def inspectable_variables
+      instance_variables - [:@type_object, :@data_object, :@blocks]
+    end
+
+
     private
 
+    # Note: this method is written this way as an optimization
+    def check_if_not_nil(a, b = nil, c = nil, d = nil, e = nil, f = nil, g = nil)
+      ok =   @data_object.has_key?(a)
+      ok &&= @data_object.has_key?(b) if b
+      ok &&= @data_object.has_key?(c) if c
+      ok &&= @data_object.has_key?(d) if d
+      ok &&= @data_object.has_key?(e) if e
+      ok &&= @data_object.has_key?(f) if f
+      ok &&= @data_object.has_key?(g) if g
+
+      if !ok
+        expected_fields = [a, b, c, d, e, f, g].compact
+        missing_fields = expected_fields.select { |x| @data_object[x].nil? }
+        raise DecodeError.new("Missing event details (#{missing_fields.map(&:to_s).join(', ')})")
+      end
+    end
+
     def self.decode_cbor_objects(data)
       objects = CBOR.decode_sequence(data)
 
@@ -92,13 +187,27 @@ module Skyfall
         raise SubscriptionError.new(data['error'], data['message'])
       end
 
-      raise DecodeError.new("Invalid object type: #{type}") unless type.is_a?(Hash)
-      raise UnsupportedError.new("Unexpected CBOR object: #{type}") unless type['op'] == 1
-      raise DecodeError.new("Missing data: #{type} #{objects.inspect}") unless type['op'] && type['t']
-      raise DecodeError.new("Invalid message type: #{type['t']}") unless type['t'].start_with?('#')
-      raise DecodeError.new("Invalid object type: #{data}") unless data.is_a?(Hash)
+      raise DecodeError.new("Invalid object type: #{type.inspect}") unless type.is_a?(Hash)
+      raise DecodeError.new("Missing data: #{type.inspect}") unless type['op'] && type['t']
+      raise DecodeError.new("Invalid object type: #{type['op'].inspect}") unless type['op'].is_a?(Integer)
+      raise DecodeError.new("Invalid object type: #{type['t'].inspect}") unless type['t'].is_a?(String)
+      raise DecodeError.new("Invalid message type: #{type['t'].inspect}") unless type['t'].start_with?('#')
+      raise UnsupportedError.new("Unsupported version: #{type['op']}") unless type['op'] == 1
+      raise DecodeError.new("Invalid object type: #{data.inspect}") unless data.is_a?(Hash)
 
       [type, data]
     end
+
+    private_class_method :decode_cbor_objects
   end
 end
+
+# need to be at the end because of a circular dependency
+
+require_relative 'account_message'
+require_relative 'commit_message'
+require_relative 'identity_message'
+require_relative 'info_message'
+require_relative 'labels_message'
+require_relative 'sync_message'
+require_relative 'unknown_message'

data/lib/skyfall/firehose/operation.rb

@@ -1,58 +1,110 @@
+# frozen_string_literal: true
+
 require_relative '../collection'
 require_relative '../firehose'
 
 module Skyfall
+
+  #
+  # A single record operation from a firehose commit event. An operation is a new record being
+  # created, or an existing record modified or deleted. It includes the URI and other details of
+  # the record in question, type of the action taken, and record data for "created" and "update"
+  # actions.
+  #
+  # Note: when a record is deleted, the previous record data is *not* included in the commit, only
+  # its URI. This means that if you're tracking records which are referencing other records, e.g.
+  # follow, block, or like records, you need to store information about this referencing record
+  # including an URI or rkey, because in case of a delete, you will not get information about which
+  # post was unliked or which account was unfollowed, only which like/follow record was deleted.
+  #
+  # At the moment, Skyfall doesn't parse the record data into any rich models specific for a given
+  # record type with a convenient API, but simply returns them as `Hash` objects (see {#raw_record}).
+  # In the future, a separate `#record` method might be added which returns a parsed record model.
+  #
+
   class Firehose::Operation
+
+    #
+    # @param message [Skyfall::Firehose::Message] commit message the operation is included in
+    # @param json [Hash] operation data
+    #
     def initialize(message, json)
       @message = message
       @json = json
     end
 
+    # @return [String] DID of the account/repository in which the operation happened
     def repo
       @message.repo
     end
 
     alias did repo
 
+    # @return [String] path part of the record URI (collection + rkey)
+    # @deprecated Use {#collection} + {#rkey}
     def path
+      @@path_warning_printed ||= false
+
+      unless @@path_warning_printed
+        $stderr.puts "Warning: Skyfall::Firehose::Operation#path is deprecated - use #collection + #rkey"
+        @@path_warning_printed = true
+      end
+
       @json['path']
     end
 
+    # @return [Symbol] type of the operation (`:create`, `:update` or `:delete`)
     def action
       @json['action'].to_sym
     end
 
+    # @return [String] record collection NSID
     def collection
       @json['path'].split('/')[0]
     end
 
+    # @return [String] record rkey
     def rkey
       @json['path'].split('/')[1]
     end
 
+    # @return [String] full AT URI of the record
     def uri
-      "at://#{repo}/#{path}"
+      "at://#{repo}/#{@json['path']}"
     end
 
+    # @return [CID, nil] CID (Content Identifier) of the record (nil for delete operations)
     def cid
       @cid ||= @json['cid'] && CID.from_cbor_tag(@json['cid'])
     end
 
+    # @return [Hash, nil] record data as a plain Ruby Hash (nil for delete operations)
     def raw_record
       @raw_record ||= @message.raw_record_for_operation(self)
     end
 
+    # Symbol short code of the collection, like `:bsky_post`. If the collection NSID is not
+    # recognized, the type is `:unknown`. The full NSID is always available through the
+    # `#collection` property.
+    #
+    # @return [Symbol]
+    # @see Skyfall::Collection
+    #
     def type
       Collection.short_code(collection)
     end
 
-    def inspectable_variables
-      instance_variables - [:@message]
-    end
-
+    # Returns a string with a representation of the object for debugging purposes.
+    # @return [String]
     def inspect
       vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
       "#<#{self.class}:0x#{object_id} #{vars}>"
     end
+
+    private
+
+    def inspectable_variables
+      instance_variables - [:@message]
+    end
   end
 end

data/lib/skyfall/firehose/sync_message.rb

@@ -1,9 +1,40 @@
+# frozen_string_literal: true
+
+require_relative '../car_archive'
 require_relative '../firehose'
+require_relative 'message'
 
 module Skyfall
+
+  #
+  # Firehose message which declares the current state of the repository. The message is meant to
+  # trigger a resynchronization of the repository from a receiving consumer, if the consumer detects
+  # from the message rev that it must have missed some events from that repository.
+  #
+  # The sync message can be emitted by a PDS or relay to force a repair of a broken account state,
+  # or e.g. when an account is created, migrated or recovered from a CAR backup.
+  #
+
   class Firehose::SyncMessage < Firehose::Message
+
+    #
+    # @private
+    # @param type_object [Hash] first decoded CBOR frame with metadata
+    # @param data_object [Hash] second decoded CBOR frame with payload
+    # @raise [DecodeError] if the message doesn't include required data
+    #
+    def initialize(type_object, data_object)
+      super
+      check_if_not_nil 'seq', 'did', 'blocks', 'rev', 'time'
+    end
+
     def rev
       @rev ||= @data_object['rev']
     end
+
+    # @return [Skyfall::CarArchive] commit data in the form of a parsed CAR archive
+    def blocks
+      @blocks ||= CarArchive.new(@data_object['blocks'])
+    end
   end
 end

data/lib/skyfall/firehose/unknown_message.rb

@@ -1,6 +1,14 @@
+# frozen_string_literal: true
+
 require_relative '../firehose'
+require_relative 'message'
 
 module Skyfall
+
+  #
+  # Firehose message of an unrecognized type.
+  #
+
   class Firehose::UnknownMessage < Firehose::Message
   end
 end

data/lib/skyfall/firehose.rb

@@ -1,9 +1,60 @@
+# frozen_string_literal: true
+
 require_relative 'stream'
 require 'uri'
 
 module Skyfall
+
+  #
+  # Client of a standard AT Protocol firehose websocket.
+  #
+  # This is the main Skyfall class to use to connect to a CBOR-based firehose
+  # websocket endpoint like `subscribeRepos` (on a PDS or a relay).
+  #
+  # To connect to the firehose, you need to:
+  #
+  # * create an instance of {Firehose}, passing it the hostname/URL of the server,
+  #   name of the endpoint (normally `:subscribe_repos`) and optionally a cursor
+  # * set up callbacks to be run when connecting, disconnecting, when a message
+  #   is received etc. (you need to set at least a message handler)
+  # * call {#connect} to start the connection
+  # * handle the received messages (instances of a {Skyfall::Firehose::Message}
+  #   subclass)
+  # 
+  # @example
+  #   client = Skyfall::Firehose.new('bsky.network', :subscribe_repos, last_cursor)
+  #   # or: client = Skyfall::Firehose.new('bsky.network', last_cursor)
+  #
+  #   client.on_message do |msg|
+  #     next unless msg.type == :commit
+  #
+  #     msg.operations.each do |op|
+  #       if op.type == :bsky_post && op.action == :create
+  #         puts "[#{msg.time}] #{msg.repo}: #{op.raw_record['text']}"
+  #       end
+  #     end
+  #   end
+  #
+  #   client.connect
+  #
+  #   # You might also want to set some or all of these lifecycle callback handlers:
+  #
+  #   client.on_connecting { |url| puts "Connecting to #{url}..." }
+  #   client.on_connect { puts "Connected" }
+  #   client.on_disconnect { puts "Disconnected" }
+  #   client.on_reconnect { puts "Connection lost, trying to reconnect..." }
+  #   client.on_timeout { puts "Connection stalled, triggering a reconnect..." }
+  #   client.on_error { |e| puts "ERROR: #{e}" }
+  #
+  # @note Most of the methods of this class that you might want to use are defined in {Skyfall::Stream}.
+  #
+
   class Firehose < Stream
+
+    # the main firehose endpoint on a PDS or relay
     SUBSCRIBE_REPOS = "com.atproto.sync.subscribeRepos"
+
+    # only used with moderation services (labellers)
     SUBSCRIBE_LABELS = "com.atproto.label.subscribeLabels"
 
     NAMED_ENDPOINTS = {
@@ -11,17 +62,68 @@ module Skyfall
       :subscribe_labels => SUBSCRIBE_LABELS
     }
 
+    # Current cursor (seq of the last seen message)
+    # @return [Integer, nil]
     attr_accessor :cursor
 
-    def initialize(server, endpoint, cursor = nil)
+    #
+    # @overload initialize(server, endpoint, cursor = nil)
+    #   Returns a new instance of a firehose client connecting to a given endpoint.
+    #
+    #   @param server [String]
+    #     Address of the server to connect to.
+    #     Expects a string with either just a hostname, or a ws:// or wss:// URL with no path.
+    #   @param endpoint [Symbol, String]
+    #     XRPC method name.
+    #     Pass either a full NSID, or a symbol shorthand from {NAMED_ENDPOINTS}
+    #   @param cursor [Integer, String, nil]
+    #     sequence number from which to resume
+    #   @raise [ArgumentError] if any of the parameters is invalid
+    #
+    # @overload initialize(server, cursor = nil)
+    #   Returns a new instance of a firehose client connecting to `subscribeRepos`.
+    #
+    #   @param server [String]
+    #     Address of the server to connect to.
+    #     Expects a string with either just a hostname, or a ws:// or wss:// URL with no path.
+    #   @param cursor [Integer, String, nil]
+    #     sequence number from which to resume
+    #   @raise [ArgumentError] if any of the parameters is invalid
+    #
+
+    def initialize(server, endpoint = nil, cursor = nil)
       require_relative 'firehose/message'
       super(server)
 
+      if cursor.nil? && (endpoint.nil? || endpoint.to_s =~ /\A\d+\z/)
+        cursor = endpoint
+        endpoint = :subscribe_repos
+      end
+
       @endpoint = check_endpoint(endpoint)
       @cursor = check_cursor(cursor)
       @root_url = ensure_empty_path(@root_url)
     end
 
+
+    protected
+
+    # Returns the full URL of the websocket endpoint to connect to.
+    # @return [String]
+
+    def build_websocket_url
+      @root_url + "/xrpc/" + @endpoint + (@cursor ? "?cursor=#{@cursor}" : "")
+    end
+
+    # Processes a single message received from the websocket. Passes the received data to the
+    # {#on_raw_message} handler, builds a {Skyfall::Firehose::Message} object, and passes it to
+    # the {#on_message} handler (if defined). Also updates the {#cursor} to this message's sequence
+    # number (note: this is skipped if {#on_message} is not set).
+    #
+    # @param msg
+    #   {https://rubydoc.info/gems/faye-websocket/Faye/WebSocket/API/MessageEvent Faye::WebSocket::API::MessageEvent}
+    # @return [nil]
+
     def handle_message(msg)
       data = msg.data
       @handlers[:raw_message]&.call(data)
@@ -38,10 +140,6 @@ module Skyfall
 
     private
 
-    def build_websocket_url
-      @root_url + "/xrpc/" + @endpoint + (@cursor ? "?cursor=#{@cursor}" : "")
-    end
-
     def check_cursor(cursor)
       if cursor.nil?
         nil

data/lib/skyfall/jetstream/account_message.rb

@@ -1,17 +1,37 @@
+# frozen_string_literal: true
+
 require_relative '../errors'
 require_relative '../jetstream'
+require_relative 'message'
 
 module Skyfall
+
+  #
+  # Jetstream message sent when the status of an account changes. This can be:
+  # 
+  # - an account being created, sending its initial state (should be active)
+  # - an account being deactivated or suspended
+  # - an account being restored back to an active state from deactivation/suspension
+  # - an account being deleted (the status returning `:deleted`)
+  #
+
   class Jetstream::AccountMessage < Jetstream::Message
+
+    #
+    # @param json [Hash] message JSON decoded from the websocket message
+    # @raise [DecodeError] if the message doesn't include required data
+    #
     def initialize(json)
-      raise DecodeError.new("Missing event details") if json['account'].nil?
+      raise DecodeError.new("Missing event details (account)") if json['account'].nil? || json['account']['active'].nil?
       super
     end
 
+    # @return [Boolean] true if the account is active, false if it's deactivated/suspended etc.
     def active?
       @json['account']['active']
     end
 
+    # @return [Symbol, nil] for inactive accounts, specifies the exact state; nil for active accounts
     def status
       @json['account']['status']&.to_sym
     end

data/lib/skyfall/jetstream/commit_message.rb

@@ -1,16 +1,49 @@
+# frozen_string_literal: true
+
 require_relative '../errors'
 require_relative '../jetstream'
+require_relative 'message'
 require_relative 'operation'
 
 module Skyfall
+
+  #
+  # Jetstream message which includes a single operation on a record in the repo (a record was
+  # created, updated or deleted). Most of the messages received from Jetstream are of this type,
+  # and this is the type you will usually be most interested in.
+  #
+
   class Jetstream::CommitMessage < Jetstream::Message
+
+    #
+    # @param json [Hash] message JSON decoded from the websocket message
+    # @raise [DecodeError] if the message doesn't include required data
+    #
     def initialize(json)
-      raise DecodeError.new("Missing event details") if json['commit'].nil?
+      raise DecodeError.new("Missing event details (commit)") if json['commit'].nil?
+
+      %w(collection rkey operation).each { |f| raise DecodeError.new("Missing event details (#{f})") if json['commit'][f].nil? }
+
       super
     end
 
+    # Returns the record operation included in the commit.
+    # @return [Jetstream::Operation]
+    #
+    def operation
+      @operation ||= Jetstream::Operation.new(self, json['commit'])
+    end
+
+    alias op operation
+
+    # Returns record operations included in the commit. Currently a `:commit` message from
+    # Jetstream always includes exactly one operation, but for compatibility with
+    # {Skyfall::Firehose}'s API it's also returned in an array here.
+    #
+    # @return [Array<Jetstream::Operation>]
+    #
     def operations
-      @operations ||= [Jetstream::Operation.new(self, json['commit'])]
+      [operation]
     end
   end
 end

data/lib/skyfall/jetstream/identity_message.rb

@@ -1,13 +1,34 @@
+# frozen_string_literal: true
+
 require_relative '../errors'
 require_relative '../jetstream'
+require_relative 'message'
 
 module Skyfall
+
+  #
+  # Jetstream message sent when a new DID is created or when the details of someone's DID document
+  # are changed (usually either a handle change or a migration to a different PDS). The message
+  # should include currently assigned handle (though the field is not required).
+  #
+  # Note: the message is originally emitted from the account's PDS and is passed as is by relays,
+  # which means you can't fully trust that the handle is actually correctly assigned to the DID
+  # and verified by DNS or well-known. To confirm that, use `DID.resolve_handle` from
+  # [DIDKit](https://ruby.sdk.blue/didkit/).
+  #
+
   class Jetstream::IdentityMessage < Jetstream::Message
+
+    #
+    # @param json [Hash] message JSON decoded from the websocket message
+    # @raise [DecodeError] if the message doesn't include required data
+    #
     def initialize(json)
-      raise DecodeError.new("Missing event details") if json['identity'].nil?
+      raise DecodeError.new("Missing event details (identity)") if json['identity'].nil?
       super
     end
 
+    # @return [String, nil] current handle assigned to the DID
     def handle
       @json['identity']['handle']
     end