skyfall 0.1.3 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: b1d8eea31c1c63f4aabd513a470083862b653f4fb1ba5a0a638da1d9e8cba3af
4
- data.tar.gz: 15e57a872800ad3a0c2a0f38be1ed31694617e82fe0981e1c9a1598c9f70e325
3
+ metadata.gz: 9a508caa504715c30f9d9ff68dbf419c8b88ad00e5fbc6315ae6a3abb624e167
4
+ data.tar.gz: 9010ece05b7f1cce0a2573ecfa75965d1dba2b77cd7926eaa8fc510f8e2ae7e2
5
5
  SHA512:
6
- metadata.gz: 84544c24ff44ed583df898567b260db3b23517de6248b105f6199de49f2a3bcfaaada701bb62252c59a44af2845321637cfa4ab5061b9f8f800d27107fb2f83e
7
- data.tar.gz: fb62308d1e517fdd257633a19f503dc9f9fce7d5b6b4d778a588c19facbe8c45ff60cca3e242ae9024c4975c9ca84326083e4eeddedd0da8d34f3ae9f647d05b
6
+ metadata.gz: b9fc0370411496e76a4caaa7b2795e999b2fbf3445e21a02e3b9135e4c7fc5e3766d7ea3a94e145becf3ab96fd4d494b0f7d8a04a09eee03698a97125d44238d
7
+ data.tar.gz: b302a5f7eefc30509a6b721809105bd9439f6919b31ab73878d132fa8a67062d0d77c17d375d3d433cb77bbba63127a823d1817f5d1a71f16c2b6601dca7d493
data/CHANGELOG.md CHANGED
@@ -1,3 +1,21 @@
1
+ ## [0.2.1] - 2023-08-19
2
+
3
+ - optimized `WebsocketMessage` parsing performance - lazy parsing of most properties (message decoding should be over 50% faster on average)
4
+ - added separate subclasses of `WebsocketMessage` for different message types
5
+ - added support for `#handle`, `#info` and `#tombstone` message types
6
+ - `UnknownMessage` is returned for unrecognized message types
7
+
8
+ ## [0.2.0] - 2023-07-24
9
+
10
+ - switched the websocket library from `websocket-client-simple` to `faye-websocket`, which should make event parsing up to ~30× faster (!)
11
+ - added `auto_reconnect` property to `Stream` (on by default) - if true, it will try to reconnect with an exponential backoff when the websocket disconnects, until you call `Stream#disconnect`
12
+
13
+ Note:
14
+
15
+ - calling `sleep` is no longer needed after connecting - call `connect` on a new thread instead to get previously default behavior of running the event loop asynchronously
16
+ - the disconnect event no longer passes an error object in the argument
17
+ - there is currently no "heartbeat" feature as in 0.1.x that checks for a stuck connection - but it doesn't seem to be needed
18
+
1
19
  ## [0.1.3] - 2023-07-04
2
20
 
3
21
  - allow passing a previously saved cursor to websocket to replay any missed events
data/README.md CHANGED
@@ -41,8 +41,6 @@ When you're ready, open the connection by calling `connect`:
41
41
  sky.connect
42
42
  ```
43
43
 
44
- The connection is started asynchronously on a separate thread. If you're running this code in a simple script (and not as a part of a server), call e.g. `sleep` without arguments or `loop { STDIN.read }` to prevent the script for exiting while the connection is open.
45
-
46
44
 
47
45
  ### Processing messages
48
46
 
data/example/firehose.rb CHANGED
@@ -21,7 +21,7 @@ end
21
21
 
22
22
  sky.on_connect { puts "Connected" }
23
23
  sky.on_disconnect { puts "Disconnected" }
24
+ sky.on_reconnect { puts "Reconnecting..." }
24
25
  sky.on_error { |e| puts "ERROR: #{e}" }
25
26
 
26
27
  sky.connect
27
- sleep
@@ -31,6 +31,9 @@ module Skyfall
31
31
  read_section(buffer) until buffer.eof?
32
32
  end
33
33
 
34
+ def section_with_cid(cid)
35
+ @sections.detect { |s| s.cid == cid }&.body
36
+ end
34
37
 
35
38
  private
36
39
 
@@ -0,0 +1,23 @@
1
+ require_relative '../car_archive'
2
+ require_relative '../cid'
3
+ require_relative '../operation'
4
+
5
+ module Skyfall
6
+ class CommitMessage < WebsocketMessage
7
+ def commit
8
+ @commit ||= @data_object['commit'] && CID.from_cbor_tag(@data_object['commit'])
9
+ end
10
+
11
+ def prev
12
+ @prev ||= @data_object['prev'] && CID.from_cbor_tag(@data_object['prev'])
13
+ end
14
+
15
+ def blocks
16
+ @blocks ||= CarArchive.new(@data_object['blocks'])
17
+ end
18
+
19
+ def operations
20
+ @operations ||= @data_object['ops'].map { |op| Operation.new(self, op) }
21
+ end
22
+ end
23
+ end
@@ -0,0 +1,7 @@
1
+ module Skyfall
2
+ class HandleMessage < WebsocketMessage
3
+ def handle
4
+ @data_object['handle']
5
+ end
6
+ end
7
+ end
@@ -0,0 +1,22 @@
1
+ module Skyfall
2
+ class InfoMessage < WebsocketMessage
3
+ attr_reader :name, :message
4
+
5
+ OUTDATED_CURSOR = "OutdatedCursor"
6
+
7
+ def initialize(type_object, data_object)
8
+ super
9
+
10
+ @name = @data_object['name']
11
+ @message = @data_object['message']
12
+ end
13
+
14
+ def to_s
15
+ (@name || "InfoMessage") + (@message ? ": #{@message}" : "")
16
+ end
17
+
18
+ def inspectable_variables
19
+ super - [:@did, :@seq]
20
+ end
21
+ end
22
+ end
@@ -0,0 +1,4 @@
1
+ module Skyfall
2
+ class TombstoneMessage < WebsocketMessage
3
+ end
4
+ end
@@ -0,0 +1,4 @@
1
+ module Skyfall
2
+ class UnknownMessage < WebsocketMessage
3
+ end
4
+ end
@@ -0,0 +1,90 @@
1
+ require_relative '../errors'
2
+ require_relative '../extensions'
3
+
4
+ require 'cbor'
5
+ require 'time'
6
+
7
+ module Skyfall
8
+ class WebsocketMessage
9
+ using Skyfall::Extensions
10
+
11
+ require_relative 'commit_message'
12
+ require_relative 'handle_message'
13
+ require_relative 'info_message'
14
+ require_relative 'tombstone_message'
15
+ require_relative 'unknown_message'
16
+
17
+ attr_reader :type_object, :data_object
18
+ attr_reader :type, :did, :seq
19
+
20
+ alias repo did
21
+
22
+ def self.new(data)
23
+ type_object, data_object = decode_cbor_objects(data)
24
+
25
+ message_class = case type_object['t']
26
+ when '#commit' then CommitMessage
27
+ when '#handle' then HandleMessage
28
+ when '#info' then InfoMessage
29
+ when '#tombstone' then TombstoneMessage
30
+ else UnknownMessage
31
+ end
32
+
33
+ message = message_class.allocate
34
+ message.send(:initialize, type_object, data_object)
35
+ message
36
+ end
37
+
38
+ def initialize(type_object, data_object)
39
+ @type_object = type_object
40
+ @data_object = data_object
41
+
42
+ @type = @type_object['t'][1..-1].to_sym
43
+ @did = @data_object['repo'] || @data_object['did']
44
+ @seq = @data_object['seq']
45
+ end
46
+
47
+ def operations
48
+ []
49
+ end
50
+
51
+ def time
52
+ @time ||= @data_object['time'] && Time.parse(@data_object['time'])
53
+ end
54
+
55
+ def inspectable_variables
56
+ instance_variables - [:@type_object, :@data_object, :@blocks]
57
+ end
58
+
59
+ def inspect
60
+ vars = inspectable_variables.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
61
+ "#<#{self.class}:0x#{object_id} #{vars}>"
62
+ end
63
+
64
+ private
65
+
66
+ def self.decode_cbor_objects(data)
67
+ objects = CBOR.decode_sequence(data)
68
+
69
+ if objects.length < 2
70
+ raise DecodeError.new("Malformed message: #{objects.inspect}")
71
+ elsif objects.length > 2
72
+ raise DecodeError.new("Invalid number of objects: #{objects.length}")
73
+ end
74
+
75
+ type, data = objects
76
+
77
+ if data['error']
78
+ raise SubscriptionError.new(data['error'], data['message'])
79
+ end
80
+
81
+ raise DecodeError.new("Invalid object type: #{type}") unless type.is_a?(Hash)
82
+ raise UnsupportedError.new("Unexpected CBOR object: #{type}") unless type['op'] == 1
83
+ raise DecodeError.new("Missing data: #{type} #{objects.inspect}") unless type['op'] && type['t']
84
+ raise DecodeError.new("Invalid message type: #{type['t']}") unless type['t'].start_with?('#')
85
+ raise DecodeError.new("Invalid object type: #{data}") unless data.is_a?(Hash)
86
+
87
+ [type, data]
88
+ end
89
+ end
90
+ end
@@ -2,30 +2,41 @@ require_relative 'collection'
2
2
 
3
3
  module Skyfall
4
4
  class Operation
5
- attr_reader :repo, :path, :action, :cid
5
+ def initialize(message, json)
6
+ @message = message
7
+ @json = json
8
+ end
6
9
 
7
- def initialize(repo, path, action, cid, record)
8
- @repo = repo
9
- @path = path
10
- @action = action.to_sym
11
- @cid = cid
12
- @record = record
10
+ def repo
11
+ @message.repo
13
12
  end
14
13
 
15
- def raw_record
16
- @record
14
+ def path
15
+ @json['path']
17
16
  end
18
17
 
19
- def uri
20
- "at://#{repo}/#{path}"
18
+ def action
19
+ @json['action'].to_sym
21
20
  end
22
21
 
23
22
  def collection
24
- path.split('/')[0]
23
+ @json['path'].split('/')[0]
25
24
  end
26
25
 
27
26
  def rkey
28
- path.split('/')[1]
27
+ @json['path'].split('/')[1]
28
+ end
29
+
30
+ def uri
31
+ "at://#{repo}/#{path}"
32
+ end
33
+
34
+ def cid
35
+ @cid ||= @json['cid'] && CID.from_cbor_tag(@json['cid'])
36
+ end
37
+
38
+ def raw_record
39
+ @raw_record ||= cid && @message.blocks.section_with_cid(cid)
29
40
  end
30
41
 
31
42
  def type
@@ -1,7 +1,8 @@
1
- require_relative 'websocket_message'
1
+ require_relative 'messages/websocket_message'
2
2
 
3
+ require 'eventmachine'
4
+ require 'faye/websocket'
3
5
  require 'uri'
4
- require 'websocket-client-simple'
5
6
 
6
7
  module Skyfall
7
8
  class Stream
@@ -11,7 +12,9 @@ module Skyfall
11
12
  :subscribe_repos => SUBSCRIBE_REPOS
12
13
  }
13
14
 
14
- attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor
15
+ MAX_RECONNECT_INTERVAL = 300
16
+
17
+ attr_accessor :heartbeat_timeout, :heartbeat_interval, :cursor, :auto_reconnect
15
18
 
16
19
  def initialize(server, endpoint, cursor = nil)
17
20
  @endpoint = check_endpoint(endpoint)
@@ -22,93 +25,71 @@ module Skyfall
22
25
  @heartbeat_interval = 5
23
26
  @heartbeat_timeout = 30
24
27
  @last_update = nil
28
+ @auto_reconnect = true
29
+ @connection_attempts = 0
25
30
  end
26
31
 
27
32
  def connect
28
- return if @websocket
33
+ return if @ws
29
34
 
30
35
  url = build_websocket_url
31
- handlers = @handlers
32
- stream = self
33
-
34
- handlers[:connecting]&.call(url)
35
-
36
- @websocket = WebSocket::Client::Simple.connect(url) do |ws|
37
- ws.on :message do |msg|
38
- stream.notify_heartbeat
39
36
 
40
- atp_message = Skyfall::WebsocketMessage.new(msg.data)
41
- stream.cursor = atp_message.seq
37
+ @handlers[:connecting]&.call(url)
38
+ @engines_on = true
42
39
 
43
- handlers[:raw_message]&.call(msg.data)
44
- handlers[:message]&.call(atp_message)
45
- end
46
-
47
- ws.on :open do
48
- handlers[:connect]&.call
40
+ EM.run do
41
+ EventMachine.error_handler do |e|
42
+ @handlers[:error]&.call(e)
49
43
  end
50
44
 
51
- ws.on :close do |e|
52
- handlers[:disconnect]&.call(e)
53
- end
45
+ @ws = Faye::WebSocket::Client.new(url)
54
46
 
55
- ws.on :error do |e|
56
- handlers[:error]&.call(e)
47
+ @ws.on(:open) do |e|
48
+ @handlers[:connect]&.call
57
49
  end
58
- end
59
50
 
60
- if @heartbeat_interval && @heartbeat_timeout && @heartbeat_thread.nil?
61
- hb_interval = @heartbeat_interval
62
- hb_timeout = @heartbeat_timeout
51
+ @ws.on(:message) do |msg|
52
+ @connection_attempts = 0
63
53
 
64
- @last_update = Time.now
54
+ data = msg.data.pack('C*')
55
+ @handlers[:raw_message]&.call(data)
65
56
 
66
- @heartbeat_thread = Thread.new do
67
- loop do
68
- sleep(hb_interval)
69
- @heartbeat_mutex.synchronize do
70
- if Time.now - @last_update > hb_timeout
71
- force_restart
72
- end
73
- end
57
+ if @handlers[:message]
58
+ atp_message = Skyfall::WebsocketMessage.new(data)
59
+ @cursor = atp_message.seq
60
+ @handlers[:message].call(atp_message)
61
+ else
62
+ @cursor = nil
74
63
  end
75
64
  end
76
- end
77
- end
78
65
 
79
- def force_restart
80
- @websocket.close
81
- @websocket = nil
66
+ @ws.on(:error) do |e|
67
+ @handlers[:error]&.call(e)
68
+ end
82
69
 
83
- timeout = 5
70
+ @ws.on(:close) do |e|
71
+ @ws = nil
84
72
 
85
- loop do
86
- begin
87
- @handlers[:reconnect]&.call
88
- connect
89
- break
90
- rescue Exception => e
91
- @handlers[:error]&.call(e)
92
- sleep(timeout)
93
- timeout *= 2
73
+ if @auto_reconnect && @engines_on
74
+ EM.add_timer(reconnect_delay) do
75
+ @connection_attempts += 1
76
+ @handlers[:reconnect]&.call
77
+ connect
78
+ end
79
+ else
80
+ @engines_on = false
81
+ @handlers[:disconnect]&.call
82
+ EM.stop_event_loop unless @ws
83
+ end
94
84
  end
95
85
  end
96
-
97
- @last_update = Time.now
98
86
  end
99
87
 
100
88
  def disconnect
101
- return unless @websocket
102
-
103
- @heartbeat_thread&.kill
104
- @heartbeat_thread = nil
89
+ return unless EM.reactor_running?
105
90
 
106
- @websocket.close
107
- @websocket = nil
108
- end
109
-
110
- def notify_heartbeat
111
- @heartbeat_mutex.synchronize { @last_update = Time.now }
91
+ @engines_on = false
92
+ EM.stop_event_loop
112
93
  end
113
94
 
114
95
  alias close disconnect
@@ -144,6 +125,14 @@ module Skyfall
144
125
 
145
126
  private
146
127
 
128
+ def reconnect_delay
129
+ if @connection_attempts == 0
130
+ 0
131
+ else
132
+ [2 ** (@connection_attempts - 1), MAX_RECONNECT_INTERVAL].min
133
+ end
134
+ end
135
+
147
136
  def build_websocket_url
148
137
  url = "wss://#{@server}/xrpc/#{@endpoint}"
149
138
  url += "?cursor=#{@cursor}" if @cursor
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  module Skyfall
4
- VERSION = "0.1.3"
4
+ VERSION = "0.2.1"
5
5
  end
data/lib/skyfall.rb CHANGED
@@ -1,5 +1,5 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require_relative 'skyfall/stream'
4
- require_relative 'skyfall/websocket_message'
4
+ require_relative 'skyfall/messages/websocket_message'
5
5
  require_relative 'skyfall/version'
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: skyfall
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.1.3
4
+ version: 0.2.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Kuba Suder
8
8
  autorequire:
9
9
  bindir: bin
10
10
  cert_chain: []
11
- date: 2023-07-04 00:00:00.000000000 Z
11
+ date: 2023-08-19 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: base32
@@ -51,25 +51,19 @@ dependencies:
51
51
  - !ruby/object:Gem::Version
52
52
  version: 0.5.9.6
53
53
  - !ruby/object:Gem::Dependency
54
- name: websocket-client-simple
54
+ name: faye-websocket
55
55
  requirement: !ruby/object:Gem::Requirement
56
56
  requirements:
57
57
  - - "~>"
58
58
  - !ruby/object:Gem::Version
59
- version: '0.6'
60
- - - ">="
61
- - !ruby/object:Gem::Version
62
- version: 0.6.1
59
+ version: 0.11.2
63
60
  type: :runtime
64
61
  prerelease: false
65
62
  version_requirements: !ruby/object:Gem::Requirement
66
63
  requirements:
67
64
  - - "~>"
68
65
  - !ruby/object:Gem::Version
69
- version: '0.6'
70
- - - ">="
71
- - !ruby/object:Gem::Version
72
- version: 0.6.1
66
+ version: 0.11.2
73
67
  description: "\n Skyfall is a Ruby library for connecting to the \"firehose\" of
74
68
  the Bluesky social network, i.e. a websocket which\n streams all new posts and
75
69
  everything else happening on the Bluesky network in real time. The code connects
@@ -92,10 +86,15 @@ files:
92
86
  - lib/skyfall/collection.rb
93
87
  - lib/skyfall/errors.rb
94
88
  - lib/skyfall/extensions.rb
89
+ - lib/skyfall/messages/commit_message.rb
90
+ - lib/skyfall/messages/handle_message.rb
91
+ - lib/skyfall/messages/info_message.rb
92
+ - lib/skyfall/messages/tombstone_message.rb
93
+ - lib/skyfall/messages/unknown_message.rb
94
+ - lib/skyfall/messages/websocket_message.rb
95
95
  - lib/skyfall/operation.rb
96
96
  - lib/skyfall/stream.rb
97
97
  - lib/skyfall/version.rb
98
- - lib/skyfall/websocket_message.rb
99
98
  - sig/skyfall.rbs
100
99
  homepage: https://github.com/mackuba/skyfall
101
100
  licenses:
@@ -1,77 +0,0 @@
1
- require_relative 'car_archive'
2
- require_relative 'cid'
3
- require_relative 'errors'
4
- require_relative 'extensions'
5
- require_relative 'operation'
6
-
7
- require 'cbor'
8
- require 'time'
9
-
10
- module Skyfall
11
- class WebsocketMessage
12
- using Skyfall::Extensions
13
-
14
- attr_reader :type_object, :data_object
15
- attr_reader :type, :repo, :time, :seq, :commit, :prev, :blocks, :operations
16
-
17
- def initialize(data)
18
- @type_object, @data_object = decode_cbor_objects(data)
19
-
20
- @type = @type_object['t'][1..-1].to_sym
21
- @operations = []
22
-
23
- @repo = @data_object['repo']
24
- @time = Time.parse(@data_object['time'])
25
- @seq = @data_object['seq']
26
-
27
- return unless @type == :commit
28
-
29
- @commit = @data_object['commit'] && CID.from_cbor_tag(@data_object['commit'])
30
- @prev = @data_object['prev'] && CID.from_cbor_tag(@data_object['prev'])
31
-
32
- @blocks = CarArchive.new(@data_object['blocks'])
33
-
34
- @operations = @data_object['ops'].map { |op|
35
- cid = op['cid'] && CID.from_cbor_tag(op['cid'])
36
- path = op['path']
37
- action = op['action']
38
- record = cid && @blocks.sections.detect { |s| s.cid == cid }.body
39
-
40
- Operation.new(@repo, path, action, cid, record)
41
- }
42
- end
43
-
44
- def inspect
45
- keys = instance_variables - [:@type_object, :@data_object, :@blocks]
46
- vars = keys.map { |v| "#{v}=#{instance_variable_get(v).inspect}" }.join(", ")
47
- "#<#{self.class}:0x#{object_id} #{vars}>"
48
- end
49
-
50
- private
51
-
52
- def decode_cbor_objects(data)
53
- objects = CBOR.decode_sequence(data)
54
-
55
- if objects.length < 2
56
- raise DecodeError.new("Malformed message: #{objects.inspect}")
57
- elsif objects.length > 2
58
- raise DecodeError.new("Invalid number of objects: #{objects.length}")
59
- end
60
-
61
- type_object, data_object = objects
62
-
63
- if data_object['error']
64
- raise SubscriptionError.new(data_object['error'], data_object['message'])
65
- end
66
-
67
- raise DecodeError.new("Invalid object type: #{type_object}") unless type_object.is_a?(Hash)
68
- raise UnsupportedError.new("Unexpected CBOR object: #{type_object}") unless type_object['op'] == 1
69
- raise DecodeError.new("Missing data: #{type_object} #{objects.inspect}") unless type_object['op'] && type_object['t']
70
- raise DecodeError.new("Invalid message type: #{type_object['t']}") unless type_object['t'].start_with?('#')
71
-
72
- raise DecodeError.new("Invalid object type: #{data_object}") unless data_object.is_a?(Hash)
73
-
74
- [type_object, data_object]
75
- end
76
- end
77
- end