skyfall 0.3.1 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1002b9dbaf48b9158f4b37d314e7f72d0990bf8290a5ad5d172af5aec9c48e38
-  data.tar.gz: 2a7cf444f30a13135e0848cbf0ad6c1900d9210e0799117e0612d5c52a93b2a4
+  metadata.gz: 1fa2648926fdc913472ec8fca4d6bdf3bbac188dbb0d9c82816538ca0c446d23
+  data.tar.gz: f35348c4834fb9dc26544105074bdc4f9ba07c36d595e5c248651d47e5dc188e
 SHA512:
-  metadata.gz: 600eaf8dd34c393b592a6490848d9e981c9ff0fa17fab629d2d640162eecda1248a49944e49d4bb679f732551e75ac209fd09bac85b7a087f81e8b13153c3446
-  data.tar.gz: e8d2c63b039aa7918337b67b77e8d0b511aba81b98d21862cf75a8561d42e72e71214fe16b11f884082fdd8b646e917b840b0369911d07afb32aedc42badc38a
+  metadata.gz: df775bdf85f5fe73f5fc5f27868b4ad8755219e98dfae72a32e7a9bb66295b463f65a2ebd9a8c956d74f200f667a9f02c00310e665a506f69a950db03eec4ad6
+  data.tar.gz: 6dfe5acf800a9765d0ad6f3ce78e7a6aae445a2dde4028ef3af39ff81a5fc3947080ecfb0ceb0dcf439a2db3e6d65e17376b187455fa1eaf56889c87d0137be1

data/CHANGELOG.md

@@ -1,6 +1,12 @@
-## [Unreleased]
+## [0.4.1] - 2024-10-04
 
-- added heartbeat timer
+- performance fix - don't decode CAR sections which aren't needed, which is most of them; this cuts the amount of memory that GC has to free up by about one third, and should speed up processing by around ~10%
+
+## [0.4.0] - 2024-09-23
+
+- (re)added a "hearbeat" feature (removed earlier in 0.2.0) to fix the occasional issue when the websocket stops receiving data, but doesn't disconnect (not enabled by default, turn it on by setting `check_heartbeat` to true)
+- added a way to set the user agent sent when connecting using the `user_agent` field (default is `"Skyfall/#{version}"`)
+- added `app.bsky.feed.postgate` record type
 
 ## [0.3.1] - 2024-06-28
 

data/LICENSE.txt

@@ -1,6 +1,6 @@
 The zlib License
 
-Copyright (c) 2023 Jakub Suder
+Copyright (c) 2023-2024 Jakub Suder
 
 This software is provided 'as-is', without any express or implied
 warranty.  In no event will the authors be held liable for any damages

data/README.md

@@ -123,9 +123,36 @@ When Skyfall receives a message about a record type that's not on the list, whet
 Do not however check if such operations have a `type` equal to `:unknown` first - just ignore the type and only check the `collection` string. The reason is that some next version of Skyfall might start recognizing those records and add a new `type` value for them like e.g. `:skygram_photo`, and then they won't match your condition anymore.
 
 
+## Configuration
+
+### User agent
+
+`Skyfall::Stream` sends a user agent header when making a connection. This is set by default to `"Skyfall/0.x.y"`, but it's recommended that you override it using the `user_agent` field to something that identifies your app and its author – this will let the owner of the server you're connecting to know who to contact in case the client is causing some problems.
+
+You can also append your user agent info to the default value like this:
+
+```rb
+sky.user_agent = "NewsBot (@news.bot) #{sky.default_user_agent}"
+```
+
+### Heartbeat and reconnecting
+
+Occasionally, especially during times of very heavy traffic, the websocket can get into a stuck state where it stops receiving any data, but doesn't disconnect and just hangs like this forever. To work around this, there is a "heartbeat" feature which starts a background timer, which periodically checks how much time has passed since the last received event, and if the time exceeds a set limit, it manually disconnects and reconnects the stream.
+
+The option is not enabled by default, because there are some firehoses which will not be sending events often, possibly only once in a while – e.g. labellers and independent PDS firehoses – and in this case we don't want any heartbeat since it will be completely normal not to have any events for a long time. It's not really possible to detect easily if we're connecting to a full network relay or one of those, so in order to avoid false alarms, you need to enable this manually using the `check_heartbeat` property.
+
+You can also change the `heartbeat_interval`, i.e. how often the timer is triggered (default: 10s), and the `heartbeat_timeout`, i.e. the amount of time passed without events when it reconnects (default: 5 min):
+
+```rb
+sky.check_heartbeat = true
+sky.heartbeat_interval = 5
+sky.heartbeat_timeout = 120
+```
+
+
 ## Credits
 
-Copyright © 2023 Kuba Suder ([@mackuba.eu](https://bsky.app/profile/mackuba.eu)).
+Copyright © 2024 Kuba Suder ([@mackuba.eu](https://bsky.app/profile/mackuba.eu)).
 
 The code is available under the terms of the [zlib license](https://choosealicense.com/licenses/zlib/) (permissive, similar to MIT).
 

data/example/follower_tracker.rb

@@ -0,0 +1,84 @@
+#!/usr/bin/env ruby
+
+# Example: track when people follow and unfollow your account.
+
+# load skyfall from a local folder - you normally won't need this
+$LOAD_PATH.unshift(File.expand_path('../lib', __dir__))
+
+require 'json'
+require 'open-uri'
+require 'skyfall'
+
+$monitored_did = ARGV[0]
+
+if $monitored_did.to_s.empty?
+  puts "Usage: #{$PROGRAM_NAME} <monitored_did>"
+  exit 1
+elsif ARGV[0] !~ /^did:plc:[a-z0-9]{24}$/
+  puts "Not a valid DID: #{$monitored_did}"
+  exit 1
+end
+
+sky = Skyfall::Stream.new('bsky.network', :subscribe_repos)
+
+sky.on_connect { puts "Connected, monitoring #{$monitored_did}" }
+sky.on_disconnect { puts "Disconnected" }
+sky.on_reconnect { puts "Reconnecting..." }
+sky.on_error { |e| puts "ERROR: #{e}" }
+
+sky.on_message do |msg|
+  # we're only interested in repo commit messages
+  next if msg.type != :commit
+
+  msg.operations.each do |op|
+    next if op.action != :create
+
+    begin
+      case op.type
+      when :bsky_block
+        process_block(msg, op)
+      when :bsky_listitem
+        process_list_item(msg, op)
+      end
+    rescue StandardError => e
+      puts "Error: #{e}"
+    end
+  end
+end
+
+def process_block(msg, op)
+  if op.raw_record['subject'] == $monitored_did
+    owner_handle = get_user_handle(op.repo)
+    puts "@#{owner_handle} has blocked you! (#{msg.time.getlocal})"
+  end
+end
+
+def process_list_item(msg, op)
+  if op.raw_record['subject'] == $monitored_did
+    owner_handle = get_user_handle(op.repo)
+
+    list_uri = op.raw_record['list']
+    list_name = get_list_name(list_uri)
+
+    puts "@#{owner_handle} has added you to list \"#{list_name}\" (#{msg.time.getlocal})"
+  end
+end
+
+def get_user_handle(did)
+  url = "https://plc.directory/#{did}"
+  json = JSON.parse(URI.open(url).read)
+  json['alsoKnownAs'][0].gsub('at://', '')
+end
+
+def get_list_name(list_uri)
+  repo, type, rkey = list_uri.gsub('at://', '').split('/')
+  url = "https://bsky.social/xrpc/com.atproto.repo.getRecord?repo=#{repo}&collection=#{type}&rkey=#{rkey}"
+
+  json = JSON.parse(URI.open(url).read)
+  json['value']['name']
+end
+
+# close the connection cleanly on Ctrl+C
+trap("SIGINT") { sky.disconnect }
+
+sky.connect

data/lib/skyfall/car_archive.rb

@@ -11,11 +11,15 @@ require 'stringio'
 
 module Skyfall
   class CarSection
-    attr_reader :cid, :body
+    attr_reader :cid
 
-    def initialize(cid, body)
+    def initialize(cid, body_data)
       @cid = cid
-      @body = body
+      @body_data = body_data
+    end
+
+    def body
+      @body ||= CarArchive.convert_data(CBOR.decode(@body_data))
     end
   end
 
@@ -37,9 +41,7 @@ module Skyfall
       section && section.body
     end
 
-    private
-
-    def convert_data(object)
+    def self.convert_data(object)
       if object.is_a?(Hash)
         object.each do |k, v|
           if v.is_a?(Hash) || v.is_a?(Array)
@@ -65,14 +67,16 @@ module Skyfall
       end
     end
 
-    def make_cid_link(cid)
+    def self.make_cid_link(cid)
       { '$link' => CID.from_cbor_tag(cid) }
     end
 
-    def make_bytes(data)
+    def self.make_bytes(data)
       { '$bytes' => Base64.encode64(data).chomp.gsub(/=+$/, '') }
     end
 
+    private
+
     def read_header(buffer)
       len = buffer.read_varint
 
@@ -112,10 +116,7 @@ module Skyfall
       cid = CID.new(prefix + cid_data)
 
       body_data = sbuffer.read
-      body = CBOR.decode(body_data)
-      convert_data(body)
-
-      @sections << CarSection.new(cid, body)
+      @sections << CarSection.new(cid, body_data)
     end
   end
 end

data/lib/skyfall/collection.rb

@@ -4,6 +4,7 @@ module Skyfall
     BSKY_FEED        = "app.bsky.feed.generator"
     BSKY_LIKE        = "app.bsky.feed.like"
     BSKY_POST        = "app.bsky.feed.post"
+    BSKY_POSTGATE    = "app.bsky.feed.postgate"
     BSKY_REPOST      = "app.bsky.feed.repost"
     BSKY_THREADGATE  = "app.bsky.feed.threadgate"
     BSKY_BLOCK       = "app.bsky.graph.block"

data/lib/skyfall/operation.rb

@@ -52,6 +52,7 @@ module Skyfall
       when Collection::BSKY_LISTBLOCK   then :bsky_listblock
       when Collection::BSKY_LISTITEM    then :bsky_listitem
       when Collection::BSKY_POST        then :bsky_post
+      when Collection::BSKY_POSTGATE    then :bsky_postgate
       when Collection::BSKY_PROFILE     then :bsky_profile
       when Collection::BSKY_REPOST      then :bsky_repost
       when Collection::BSKY_STARTERPACK then :bsky_starterpack

data/lib/skyfall/stream.rb

@@ -14,11 +14,12 @@ module Skyfall
       :subscribe_labels => SUBSCRIBE_LABELS
     }
 
-    EVENTS = %w(message raw_message connecting connect disconnect reconnect error)
+    EVENTS = %w(message raw_message connecting connect disconnect reconnect error timeout)
 
     MAX_RECONNECT_INTERVAL = 300
 
-    attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor, :auto_reconnect
+    attr_accessor :cursor, :auto_reconnect, :last_update, :user_agent
+    attr_accessor :heartbeat_timeout, :heartbeat_interval, :check_heartbeat
 
     def initialize(server, endpoint, cursor = nil)
       @endpoint = check_endpoint(endpoint)
@@ -26,7 +27,12 @@ module Skyfall
       @cursor = check_cursor(cursor)
       @handlers = {}
       @auto_reconnect = true
+      @check_heartbeat = false
       @connection_attempts = 0
+      @heartbeat_interval = 10
+      @heartbeat_timeout = 300
+      @last_update = nil
+      @user_agent = default_user_agent
 
       @handlers[:error] = proc { |e| puts "ERROR: #{e}" }
     end
@@ -47,15 +53,18 @@ module Skyfall
           @handlers[:error]&.call(e)
         end
 
-        @ws = Faye::WebSocket::Client.new(url)
+        @ws = Faye::WebSocket::Client.new(url, nil, { headers: { 'User-Agent' => user_agent }})
 
         @ws.on(:open) do |e|
           @handlers[:connect]&.call
+          @last_update = Time.now
+          start_heartbeat_timer
         end
 
         @ws.on(:message) do |msg|
           @reconnecting = false
           @connection_attempts = 0
+          @last_update = Time.now
 
           data = msg.data.pack('C*')
           @handlers[:raw_message]&.call(data)
@@ -85,6 +94,7 @@ module Skyfall
               connect
             end
           else
+            stop_heartbeat_timer
             @engines_on = false
             @handlers[:disconnect]&.call
             EM.stop_event_loop unless @ws
@@ -110,6 +120,40 @@ module Skyfall
 
     alias close disconnect
 
+    def default_user_agent
+      "Skyfall/#{Skyfall::VERSION}"
+    end
+
+    def check_heartbeat=(value)
+      @check_heartbeat = value
+
+      if @check_heartbeat && @engines_on && @ws && !@heartbeat_timer
+        start_heartbeat_timer
+      elsif !@check_heartbeat && @heartbeat_timer
+        stop_heartbeat_timer
+      end
+    end
+
+    def start_heartbeat_timer
+      return if !@check_heartbeat || @heartbeat_interval.to_f <= 0 || @heartbeat_timeout.to_f <= 0
+      return if @heartbeat_timer
+
+      @heartbeat_timer = EM::PeriodicTimer.new(@heartbeat_interval) do
+        next if @ws.nil? || @heartbeat_timeout.to_f <= 0
+        time_passed = Time.now - @last_update
+
+        if time_passed > @heartbeat_timeout
+          @handlers[:timeout]&.call
+          reconnect
+        end
+      end
+    end
+
+    def stop_heartbeat_timer
+      @heartbeat_timer&.cancel
+      @heartbeat_timer = nil
+    end
+
     EVENTS.each do |event|
       define_method "on_#{event}" do |&block|
         @handlers[event.to_sym] = block

data/lib/skyfall/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Skyfall
-  VERSION = "0.3.1"
+  VERSION = "0.4.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: skyfall
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.4.1
 platform: ruby
 authors:
 - Kuba Suder
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-28 00:00:00.000000000 Z
+date: 2024-10-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: base32
@@ -114,6 +114,7 @@ files:
 - LICENSE.txt
 - README.md
 - example/block_tracker.rb
+- example/follower_tracker.rb
 - example/monitor_phrases.rb
 - example/print_all_posts.rb
 - example/push_notifications.rb