tempest-rb 0.0.1

data/lib/tempest/handle_resolver.rb

@@ -0,0 +1,42 @@
+require_relative "../tempest"
+
+module Tempest
+  # Resolves AT Protocol DIDs to Bluesky handles via app.bsky.actor.getProfile.
+  # Caches both positive and negative lookups in-process so a busy Jetstream
+  # feed doesn't hammer the PDS on every event.
+  class HandleResolver
+    NOT_FOUND = Object.new.freeze
+
+    def initialize(client:)
+      @client = client
+      @cache = {}
+      @mutex = Mutex.new
+    end
+
+    def resolve(did)
+      cached = @mutex.synchronize { @cache[did] }
+      return cached_value(cached) unless cached.nil?
+
+      handle = lookup(did)
+      @mutex.synchronize { @cache[did] = handle.nil? ? NOT_FOUND : handle }
+      handle
+    end
+
+    def seed(did, handle)
+      @mutex.synchronize { @cache[did] = handle }
+    end
+
+    private
+
+    def cached_value(value)
+      value.equal?(NOT_FOUND) ? nil : value
+    end
+
+    def lookup(did)
+      response = @client.get("app.bsky.actor.getProfile", query: { "actor" => did })
+      response.is_a?(Hash) ? response["handle"] : nil
+    rescue Tempest::APIError
+      nil
+    end
+  end
+end

data/lib/tempest/http.rb

@@ -0,0 +1,88 @@
+require "json"
+require "uri"
+require "async"
+require "async/http/internet"
+
+require_relative "../tempest"
+
+module Tempest
+  # JSON-over-HTTP transport for XRPC endpoints.
+  # Backed by Async::HTTP::Internet, which keeps connections alive per origin
+  # and reuses them across calls. The public interface stays synchronous
+  # (returns Response on call) by wrapping work in Sync so the REPL doesn't
+  # need to know about Async.
+  module HTTP
+    Response = Struct.new(:status, :body) do
+      def ok?
+        status >= 200 && status < 300
+      end
+
+      def unauthorized?
+        status == 401
+      end
+    end
+
+    @internet_mutex = Mutex.new
+    @internet = nil
+
+    module_function
+
+    def post_json(url, body: nil, headers: {})
+      request("POST", url, body: body, headers: headers)
+    end
+
+    def get_json(url, headers: {}, query: nil)
+      uri = URI(url)
+      if query && !query.empty?
+        existing = uri.query ? URI.decode_www_form(uri.query) : []
+        uri.query = URI.encode_www_form(existing + query.to_a)
+      end
+      request("GET", uri.to_s, headers: headers)
+    end
+
+    def request(method, url, body: nil, headers: {})
+      normalized = headers.each_with_object({}) { |(k, v), h| h[k.to_s.downcase] = v }
+      payload = nil
+      if body
+        normalized["content-type"] ||= "application/json"
+        payload = [JSON.generate(body)]
+      end
+      normalized["accept"] ||= "application/json"
+
+      header_pairs = normalized.to_a
+
+      Sync do
+        response = internet.call(method, url, header_pairs, payload)
+        begin
+          body_str = response.read.to_s
+          Response.new(response.status, parse_body(response, body_str))
+        ensure
+          response.close
+        end
+      end
+    end
+
+    def parse_body(response, body_str)
+      return nil if body_str.empty?
+      ctype = response.headers["content-type"].to_s
+      return JSON.parse(body_str) if ctype.include?("application/json")
+      body_str
+    end
+
+    def internet
+      @internet_mutex.synchronize do
+        @internet ||= Async::HTTP::Internet.new
+      end
+    end
+
+    def reset!
+      @internet_mutex.synchronize do
+        existing = @internet
+        @internet = nil
+        if existing
+          Sync { existing.close }
+        end
+      end
+    end
+  end
+end

data/lib/tempest/id_var.rb

@@ -0,0 +1,44 @@
+require_relative "../tempest"
+
+module Tempest
+  # Earthquake-style identifier ring. Each generator owns a fixed list of
+  # slots (e.g. "AA".."ZZ" = 676 slots). New ids consume the next slot;
+  # when the ring wraps the previous tenant of the recycled slot is
+  # evicted from both the forward (id => var) and reverse (var => id)
+  # tables so callers never see a stale mapping.
+  #
+  # Not thread-safe. The REPL renders posts on a single thread (either
+  # the main REPL thread or behind Screen's mutex) so external
+  # serialization is sufficient.
+  class IdVar
+    def initialize(range:, prefix: "$")
+      @slots = range.to_a
+      raise ArgumentError, "range produced no slots" if @slots.empty?
+      @prefix = prefix
+      @cursor = -1
+      @forward = {} # id => var
+      @reverse = {} # var => id
+    end
+
+    def generate(id)
+      return @forward[id] if @forward.key?(id)
+      @cursor = (@cursor + 1) % @slots.length
+      var = "#{@prefix}#{@slots[@cursor]}"
+      evict(var)
+      @forward[id] = var
+      @reverse[var] = id
+      var
+    end
+
+    def lookup(var)
+      @reverse[var]
+    end
+
+    private
+
+    def evict(var)
+      old_id = @reverse.delete(var)
+      @forward.delete(old_id) if old_id
+    end
+  end
+end

data/lib/tempest/jetstream/client.rb

@@ -0,0 +1,69 @@
+require "uri"
+
+require_relative "../../tempest"
+require_relative "decoder"
+
+module Tempest
+  module Jetstream
+    DEFAULT_URL = "wss://jetstream2.us-east.bsky.network/subscribe".freeze
+
+    class Client
+      def initialize(url: DEFAULT_URL, wanted_collections: [], wanted_dids: [], decoder: Decoder, transport: nil)
+        @url = url
+        @wanted_collections = Array(wanted_collections)
+        @wanted_dids = Array(wanted_dids)
+        @decoder = decoder
+        @transport = transport
+      end
+
+      def subscribe_url(cursor: nil)
+        params = []
+        @wanted_collections.each { |c| params << ["wantedCollections", c] }
+        @wanted_dids.each { |d| params << ["wantedDids", d] }
+        params << ["cursor", cursor.to_s] if cursor
+        return @url if params.empty?
+
+        uri = URI(@url)
+        existing = uri.query ? URI.decode_www_form(uri.query) : []
+        uri.query = URI.encode_www_form(existing + params)
+        uri.to_s
+      end
+
+      def each_event(cursor: nil, &block)
+        return enum_for(:each_event, cursor: cursor) unless block
+
+        transport.each_message(subscribe_url(cursor: cursor)) do |raw|
+          event = @decoder.decode(raw)
+          yield event if event
+        end
+      end
+
+      private
+
+      def transport
+        @transport ||= AsyncWebSocketTransport.new
+      end
+    end
+
+    # Default WebSocket transport using async-websocket. Loaded lazily so unit
+    # tests that inject a stub transport don't pull in the Async runtime.
+    class AsyncWebSocketTransport
+      def initialize
+        require "async"
+        require "async/http/endpoint"
+        require "async/websocket/client"
+      end
+
+      def each_message(url)
+        Async do |task|
+          endpoint = Async::HTTP::Endpoint.parse(url)
+          Async::WebSocket::Client.connect(endpoint) do |connection|
+            while (message = connection.read)
+              yield message.buffer
+            end
+          end
+        end.wait
+      end
+    end
+  end
+end

data/lib/tempest/jetstream/decoder.rb

@@ -0,0 +1,81 @@
+require "json"
+
+require_relative "../../tempest"
+require_relative "../facet"
+
+module Tempest
+  module Jetstream
+    Event = Data.define(
+      :kind,
+      :did,
+      :time_us,
+      :collection,
+      :operation,
+      :rkey,
+      :cid,
+      :text,
+      :created_at,
+      :subject_uri,
+      :facets,
+      :reply_parent_uri,
+    ) do
+      def initialize(kind:, did:, time_us:, collection:, operation:, rkey:, cid:,
+                     text:, created_at:, subject_uri: nil, facets: [],
+                     reply_parent_uri: nil)
+        super
+      end
+
+      def post?
+        collection == "app.bsky.feed.post"
+      end
+
+      def like?
+        collection == "app.bsky.feed.like"
+      end
+
+      def repost?
+        collection == "app.bsky.feed.repost"
+      end
+
+      def create?
+        operation == :create
+      end
+
+      def at_uri
+        "at://#{did}/#{collection}/#{rkey}"
+      end
+    end
+
+    module Decoder
+      module_function
+
+      def decode(payload)
+        message = JSON.parse(payload)
+        return nil unless message["kind"] == "commit"
+
+        commit = message["commit"] || {}
+        record = commit["record"] || {}
+        subject = record["subject"]
+        reply = record["reply"]
+        reply_parent = reply.is_a?(Hash) ? reply["parent"] : nil
+
+        Event.new(
+          kind: :commit,
+          did: message["did"],
+          time_us: message["time_us"],
+          collection: commit["collection"],
+          operation: commit["operation"]&.to_sym,
+          rkey: commit["rkey"],
+          cid: commit["cid"],
+          text: record["text"],
+          created_at: record["createdAt"],
+          subject_uri: subject.is_a?(Hash) ? subject["uri"] : nil,
+          facets: Tempest::Facet.parse(record["facets"]),
+          reply_parent_uri: reply_parent.is_a?(Hash) ? reply_parent["uri"] : nil,
+        )
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/tempest/jetstream/stream_manager.rb

@@ -0,0 +1,262 @@
+require_relative "../../tempest"
+require_relative "../debug_log"
+require_relative "client"
+
+module Tempest
+  module Jetstream
+    # Runs a Jetstream::Client in a background thread so the REPL stays
+    # responsive. The transport itself is fiber-based, but we keep that fiber
+    # off the main thread to avoid interleaving with Reline's blocking read.
+    # Owns reconnect-with-cursor so a flaky socket or sleep/wake cycle doesn't
+    # silently strand the live feed.
+    class StreamManager
+      DEFAULT_BACKOFF = [1, 2, 5, 10, 30].freeze
+      # Conservative replay window: Jetstream's default event-ttl is 24h, but
+      # Bluesky doesn't publicly commit to that for their hosted instances and
+      # boundary cases (clock skew, tail trim races) bite around the limit. If
+      # we've been offline longer than this, drop the cursor and let the Runner
+      # backfill via getTimeline.
+      CURSOR_WINDOW_SECONDS = 12 * 60 * 60
+      # How often we persist the cursor during a stable live-tail. 5s caps the
+      # worst-case event loss on crash to a few seconds of activity while
+      # keeping disk writes negligible on a busy stream.
+      DEFAULT_CURSOR_SAVE_INTERVAL = 5.0
+
+      def initialize(client:, backoff: DEFAULT_BACKOFF, sleeper: ->(s) { sleep(s) },
+                     clock: -> { Time.now }, cursor_store: nil,
+                     cursor_save_interval: DEFAULT_CURSOR_SAVE_INTERVAL,
+                     filter: nil, logger: nil)
+        @client = client
+        @backoff = backoff
+        @sleeper = sleeper
+        @clock = clock
+        @cursor_store = cursor_store
+        @cursor_save_interval = cursor_save_interval
+        @filter = filter
+        @logger = logger || Tempest::DebugLog.build_null_logger
+        @thread = nil
+        @mutex = Mutex.new
+        @stopping = false
+        @cursor_state = { live: nil, saved: nil }
+        @last_event_at = nil
+      end
+
+      def start(&on_event)
+        @mutex.synchronize do
+          return if @thread&.alive?
+          @stopping = false
+          @thread = Thread.new { run(on_event) }
+        end
+      end
+
+      def stop
+        @logger.info("stream") do
+          live = @mutex.synchronize { @cursor_state[:live] }
+          "stopping final_cursor=#{live.inspect}"
+        end
+        @mutex.synchronize { @stopping = true }
+        thread = @mutex.synchronize do
+          t = @thread
+          @thread = nil
+          t
+        end
+        thread&.kill
+        thread&.join
+        flush_cursor!
+      end
+
+      def running?
+        @mutex.synchronize { !!@thread&.alive? }
+      end
+
+      # Time of the last event yielded by the underlying client, regardless of
+      # whether the filter accepted it. Watchdog reads this to detect a stalled
+      # socket (kernel still thinks the TCP connection is alive but no bytes
+      # are arriving).
+      def last_event_at
+        @mutex.synchronize { @last_event_at }
+      end
+
+      # Break a stalled each_event so the reconnect loop can run. Used by the
+      # Watchdog when the kernel hasn't surfaced the disconnect (e.g., after
+      # macOS sleep/wake). Safe to call from another thread or when no worker
+      # is running.
+      def force_reconnect
+        thread = @mutex.synchronize { @thread }
+        return unless thread&.alive?
+        @logger.warn("stream") { "force_reconnect requested" }
+        begin
+          thread.raise(Stalled.new("forced reconnect"))
+        rescue ThreadError
+          # Thread already exited between alive? and raise — nothing to do.
+        end
+      end
+
+      private
+
+      def run(on_event)
+        Thread.current.report_on_exception = false
+        cursor, startup_gap_since = load_initial_cursor
+        if startup_gap_since
+          @logger.warn("stream") { "startup_stale stale_since=#{startup_gap_since.iso8601}" }
+          on_event.call(StreamStatus.new(state: :gapped, since: startup_gap_since))
+        end
+        last_saved_cursor = cursor
+        last_save_at = nil
+        attempt = 0
+
+        @logger.info("stream") do
+          age = cursor ? cursor_age_seconds(cursor) : nil
+          "worker start cursor=#{cursor.inspect} cursor_age_seconds=#{age.inspect}"
+        end
+
+        until stopping?
+          # Detect a long offline gap from the cursor's age rather than from
+          # wall-clock disconnect timestamps. When the host machine sleeps,
+          # the background thread is suspended and we only learn about the
+          # outage at wake time — `disconnected_at` would therefore reflect
+          # the wake time, not the actual go-offline time, and the window
+          # check would never fire. The cursor (a unix-microseconds event
+          # timestamp from Jetstream) is unaffected by our suspension, so its
+          # age is a reliable proxy for "how long since we last saw events".
+          if attempt > 0 && cursor
+            cursor_age = @clock.call.to_f - (cursor / 1_000_000.0)
+            if cursor_age > CURSOR_WINDOW_SECONDS
+              since = Time.at(cursor / 1_000_000.0)
+              @logger.warn("stream") { "gapped cursor_age_seconds=#{cursor_age.round(1)} since=#{since.iso8601}" }
+              on_event.call(StreamStatus.new(state: :gapped, since: since))
+              cursor = nil
+            end
+          end
+
+          if attempt > 0
+            delay = @backoff[[attempt - 1, @backoff.length - 1].min]
+            @logger.info("stream") { "reconnecting attempt=#{attempt} cursor=#{cursor.inspect} backoff_just_slept=#{delay}" }
+            on_event.call(StreamStatus.new(state: :reconnecting))
+          end
+
+          error = nil
+          saw_event = false
+          @logger.info("stream") { "subscribe cursor=#{cursor.inspect}" }
+          begin
+            @client.each_event(cursor: cursor) do |event|
+              now = @clock.call
+              @mutex.synchronize { @last_event_at = now }
+              if event.respond_to?(:time_us) && event.time_us
+                cursor = event.time_us
+                @mutex.synchronize { @cursor_state[:live] = cursor }
+                if @cursor_store && cursor != last_saved_cursor
+                  if last_save_at.nil? || (now - last_save_at) >= @cursor_save_interval
+                    @cursor_store.save(time_us: cursor, at: now)
+                    @logger.debug("stream") { "cursor save time_us=#{cursor}" }
+                    last_saved_cursor = cursor
+                    last_save_at = now
+                    @mutex.synchronize { @cursor_state[:saved] = cursor }
+                  end
+                end
+              end
+              next if @filter && !@filter.call(event)
+
+              if attempt > 0 && !saw_event
+                on_event.call(StreamStatus.new(state: :live))
+              end
+              saw_event = true
+              on_event.call(event)
+            end
+          rescue Stalled => e
+            error = e
+            @logger.warn("stream") { "stalled — forced reconnect cursor=#{cursor.inspect}" }
+            on_event.call(StreamError.new(e))
+          rescue => e
+            error = e
+            @logger.warn("stream") { "disconnect error=#{e.class}: #{e.message}" }
+            on_event.call(StreamError.new(e))
+          end
+
+          break if stopping?
+
+          # Force a final save on disconnect so we don't lose the tail between
+          # the throttle interval and the connection drop.
+          if @cursor_store && cursor && cursor != last_saved_cursor
+            now = @clock.call
+            @cursor_store.save(time_us: cursor, at: now)
+            @logger.debug("stream") { "cursor save (disconnect) time_us=#{cursor}" }
+            last_saved_cursor = cursor
+            last_save_at = now
+            @mutex.synchronize { @cursor_state[:saved] = cursor }
+          end
+
+          on_event.call(
+            StreamStatus.new(
+              state: :disconnected,
+              reason: error ? :error : :closed,
+              error: error,
+            ),
+          )
+
+          delay = @backoff[[attempt, @backoff.length - 1].min]
+          @sleeper.call(delay)
+          attempt += 1
+        end
+
+        @logger.info("stream") { "worker exit final_cursor=#{cursor.inspect}" }
+      end
+
+      def cursor_age_seconds(cursor)
+        return nil unless cursor
+        (@clock.call - Time.at(cursor / 1_000_000.0)).round(1)
+      rescue StandardError
+        nil
+      end
+
+      def stopping?
+        @mutex.synchronize { @stopping }
+      end
+
+      # Returns [cursor, gap_since]. `gap_since` is non-nil when a persisted
+      # cursor existed but is too old to replay safely; the caller emits
+      # :gapped (so the Runner backfills via getTimeline) and subscribes
+      # without a cursor.
+      def load_initial_cursor
+        return [nil, nil] unless @cursor_store
+        stored = @cursor_store.load
+        return [nil, nil] unless stored && stored[:time_us] && stored[:saved_at]
+        age = @clock.call - stored[:saved_at]
+        return [nil, stored[:saved_at]] if age > CURSOR_WINDOW_SECONDS
+        [stored[:time_us], nil]
+      end
+
+      # Called from `stop` after the worker thread has been killed. Ensures the
+      # most recent in-memory cursor (which the throttle may have skipped over)
+      # makes it to disk; otherwise a crash during a stable live-tail would
+      # roll us back by `cursor_save_interval` worth of events on next launch.
+      def flush_cursor!
+        return unless @cursor_store
+        live, saved = @mutex.synchronize { [@cursor_state[:live], @cursor_state[:saved]] }
+        return unless live && live != saved
+        @cursor_store.save(time_us: live, at: @clock.call)
+        @mutex.synchronize { @cursor_state[:saved] = live }
+      end
+    end
+
+    # Raised inside the worker thread by StreamManager#force_reconnect to
+    # break a stalled each_event. The run loop catches it and treats it as a
+    # disconnect, preserving the existing reconnect-with-cursor flow.
+    class Stalled < StandardError; end
+
+    StreamError = Struct.new(:cause)
+
+    # Lifecycle status emitted alongside Event/StreamError on the same
+    # on_event callback so the REPL can render "-- disconnected" /
+    # "-- reconnecting" / "-- live" lines without coupling to the manager's
+    # internals. `state` is one of :disconnected | :reconnecting | :live |
+    # :gapped. `reason` is :closed | :error for :disconnected. `error` is the
+    # underlying exception when reason == :error. `since` is the disconnect
+    # time when state == :gapped.
+    StreamStatus = Data.define(:state, :reason, :error, :since) do
+      def initialize(state:, reason: nil, error: nil, since: nil)
+        super
+      end
+    end
+  end
+end

data/lib/tempest/jetstream/subscription.rb

@@ -0,0 +1,34 @@
+require "set"
+
+require_relative "../../tempest"
+
+module Tempest
+  module Jetstream
+    # Decides whether the Jetstream subscription can use server-side wantedDids
+    # filtering or has to fall back to a firehose-plus-client-side-filter
+    # arrangement. Jetstream caps wantedDids at 10000 DIDs per subscription, so
+    # anyone following more than that has to receive the full stream and drop
+    # uninteresting events locally.
+    Plan = Data.define(:wanted_dids, :filter)
+
+    module Subscription
+      module_function
+
+      def build(self_did:, follows:, cap: 10_000)
+        ordered = [self_did]
+        follows.each do |row|
+          did = row[:did] || row["did"]
+          next if did.nil? || did == self_did
+          ordered << did
+        end
+
+        if ordered.length <= cap
+          Plan.new(wanted_dids: ordered, filter: nil)
+        else
+          allowed = ordered.to_set
+          Plan.new(wanted_dids: [], filter: ->(event) { allowed.include?(event.did) })
+        end
+      end
+    end
+  end
+end