mtproto 0.0.27 → 0.0.29

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c5077ec3124cfdacfdadca11188ff9a028ceb982a69e85ab78b05f4d9847430e
-  data.tar.gz: be1a48d79d0f0f36bd032ceaec9df07ab2ecc9c3f3a0dccd2029b75083f2cddb
+  metadata.gz: 0a50a3572643e17c2435e3b310979e9a9ddbbf498bf8febb5819d96a73443899
+  data.tar.gz: 622d75782346d1cf190e3daa6097305afc3520b5fe5834b727a79bf4c0e86b36
 SHA512:
-  metadata.gz: 4b7d8d18e86aca190aef0f7309f7f1235e926035ac62858efb429700d19d701a16465f7cd34fc1a94a49eb845739da10a9fef4579b4d98ab95497c0c52e3afa7
-  data.tar.gz: ece9d015beeadd70fdf95422ec924aad10d0f585b0c9a7ac2e51f8d415069703e05d065426451fdb33bbebe8b90e33b5fd027acd4ebc6fdf162d2fdaaeb325ff
+  metadata.gz: 7ec7ee0beac3ee1143720d8e4972dba6a6f7db379069b0cc2a3274b5c81becb8e1353f448654b4c6c9d0385472d4985725292da56c040a50476ec94d2f980a93
+  data.tar.gz: c0951ce0991d6c960f6963647849156333bf5bebe4d59b0857af6703707430249c54bc4805be7b8e2527df84141f68dce9c43e6668ca5200eb4f4b7f3d146d4c

data/lib/mtproto/client/api/get_channel_difference.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../tl/objects/get_channel_difference'
 require_relative '../../tl/objects/channel_difference'
+require_relative '../../tl/objects/raw_response'
 
 module MTProto
   class Client
@@ -17,6 +18,13 @@ module MTProto
           TL::ChannelDifference
         ).body
       end
+
+      # The raw getChannelDifference bytes, for the update state machine (which parses
+      # the reply into its rich event shape). Goes through rpc_call for FLOOD_WAIT
+      # retry and CONNECTION_NOT_INITED reinit, same as get_channel_difference.
+      def get_channel_difference_raw(channel:, pts:, limit: 100)
+        rpc_call(TL::GetChannelDifference.new(channel: channel, pts: pts, limit: limit), TL::RawResponse).body.raw_bytes
+      end
     end
   end
 end

data/lib/mtproto/client/api/get_updates_difference.rb

@@ -2,6 +2,7 @@
 
 require_relative '../../tl/objects/get_difference'
 require_relative '../../tl/objects/updates_difference'
+require_relative '../../tl/objects/raw_response'
 
 module MTProto
   class Client
@@ -16,6 +17,15 @@ module MTProto
           TL::UpdatesDifference
         ).body
       end
+
+      # The raw getDifference bytes, for a caller that parses the reply itself (the
+      # update state machine, which needs the rich event shape TL::UpdatesDifference
+      # does not surface). Goes through rpc_call, so it inherits FLOOD_WAIT retry and
+      # CONNECTION_NOT_INITED reinit — both common while draining a backlog / after a
+      # reconnect.
+      def get_updates_difference_raw(pts:, date:, qts:)
+        rpc_call(TL::GetDifference.new(pts: pts, date: date, qts: qts), TL::RawResponse).body.raw_bytes
+      end
     end
   end
 end

data/lib/mtproto/client/update_api.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+require_relative '../updates'
+
+module MTProto
+  class Client
+    # The high-level update-consumption surface of a Client, mixed into it: register
+    # update callbacks (with a supported replace/clear seam instead of reaching into
+    # the callback list), the post-reconnect hook, and run_updates — the normalized,
+    # ordered, exactly-once event stream. The low-level fan-out (dispatch_update) and
+    # the @on_update_callbacks / @on_reconnect state stay in Client.
+    module UpdateApi
+      def on_update(&block)
+        @on_update_callbacks << block
+      end
+
+      # Replace every registered update callback with a single one, so a
+      # re-registration never stacks handlers (run_updates relies on this).
+      def replace_update_callback(&block)
+        @on_update_callbacks = [block]
+      end
+
+      def clear_update_callbacks
+        @on_update_callbacks = []
+      end
+
+      # Register a callback fired after the transport reconnects (a known gap point).
+      # It runs on the receiver fiber, so it must NOT block — enqueue work instead.
+      def on_reconnect(&block)
+        @on_reconnect = block
+      end
+
+      # High-level update consumption: hand the block a normalized, ordered,
+      # de-duplicated stream of Updates::Event, with the account and per-channel
+      # positions read/written through `store` (see Updates::MemoryStore for the
+      # contract). on_start, if given, runs once inside the started mainloop before
+      # the first update is consumed — where host startup that needs the API belongs,
+      # since an API call raises until the mainloop is running. Blocks until the client
+      # shuts down; Updates::Runner owns the wiring.
+      def run_updates(store:, logger: nil, on_start: nil, &)
+        Updates::Runner.new(client: self, store: store, logger: logger).run(on_start: on_start, &)
+      end
+    end
+  end
+end

data/lib/mtproto/client.rb

@@ -5,6 +5,7 @@ require 'digest'
 require 'base64'
 require 'async'
 require 'async/condition'
+require_relative 'updates'
 require_relative 'tl/constructors'
 require_relative 'transport/tcp_connection'
 require_relative 'transport/connection'
@@ -12,6 +13,7 @@ require_relative 'transport/abridged_packet_codec'
 require_relative 'tl/objects/message'
 require_relative 'client/rpc'
 require_relative 'client/api'
+require_relative 'client/update_api'
 require_relative 'tl/objects/invoke_with_layer'
 require_relative 'tl/objects/init_connection'
 require_relative 'tl/objects/get_config'
@@ -20,6 +22,7 @@ require_relative 'tl/objects/help_config'
 module MTProto
   class Client
     extend DelegateMethods
+    include UpdateApi
 
     API_LAYER = 227
 
@@ -82,6 +85,7 @@ module MTProto
       @ack_ids = []
       @running = false
       @on_update_callbacks = []
+      @on_reconnect = nil
     end
 
     # Keepalive cadence: flush pending acks every ACK_INTERVAL seconds, send a
@@ -94,10 +98,6 @@ module MTProto
     RECONNECT_BACKOFF_BASE = 1
     RECONNECT_BACKOFF_MAX = 30
 
-    def on_update(&block)
-      @on_update_callbacks << block
-    end
-
     def run_mainloop
       raise 'Auth key not set' unless auth_key?
       raise 'Mainloop already running' if @running
@@ -288,6 +288,7 @@ module MTProto
       return unless @running
 
       Async { init_connection! }
+      @on_reconnect&.call
     end
 
     # Reconnect the transport, retrying with exponential backoff until it succeeds

data/lib/mtproto/updates/event.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module MTProto
+  module Updates
+    # One normalized update the state machine emits, unified across the live push
+    # and the getDifference/getChannelDifference recovery paths so a host sees one
+    # shape however an update arrived. `kind` selects which payload fields are set:
+    #   :message / :edit          message (TL::Message#to_h)
+    #   :join                     joined_user_id
+    #   :leave                    left_user_id
+    #   :reaction / :unreaction   msg_id, reactors
+    #   :delete                   deleted_ids
+    #   :callback                 query_id, user_id, data, msg_id
+    #   :guest_query              query_id, message, qts
+    # `pts`/`pts_count`/`channel` drive ordering and gap detection; they are nil for
+    # updates outside the pts sequence (e.g. getDifference's common new_messages,
+    # which the protocol delivers without a per-message pts). `users`/`chats` are the
+    # peers that rode in the same batch (shared by reference) — the access_hash source
+    # a host caches and enriches from.
+    Event = Struct.new(
+      :kind, :channel, :pts, :pts_count,
+      :peer_type, :peer_id, :msg_id,
+      :message, :joined_user_id, :left_user_id, :deleted_ids, :reactors,
+      :query_id, :user_id, :data, :qts,
+      :users, :chats,
+      keyword_init: true
+    )
+  end
+end

data/lib/mtproto/updates/event_builder.rb

@@ -0,0 +1,286 @@
+# frozen_string_literal: true
+
+require_relative '../tl/reader'
+require_relative '../tl/objects/message'
+require_relative '../tl/objects/updates'
+require_relative 'event'
+
+module MTProto
+  module Updates
+    # Turns ONE Telegram Update (a single entry of a Vector<Update>, or the inner
+    # update of an updateShort) into a normalized Event, or nil for an update we
+    # don't surface. Reuses the gem's TL::Message / Reader / Schema and the existing
+    # TL::Updates callback/guest-query readers. The envelope walking (which vector,
+    # which reply) lives in Parser; this module owns the per-update semantics.
+    module EventBuilder
+      module_function
+
+      UPDATE_NEW_MESSAGE = 0x1f2b0afd
+      UPDATE_NEW_CHANNEL_MESSAGE = 0x62ba04d9
+      UPDATE_EDIT_MESSAGE = 0xe40370a3
+      UPDATE_EDIT_CHANNEL_MESSAGE = 0x1b3f4df7
+      UPDATE_MESSAGE_REACTIONS = 0x1e297bfa
+      UPDATE_BOT_MESSAGE_REACTION = 0xac21d3ce
+      UPDATE_DELETE_CHANNEL_MESSAGES = 0xc32d5b12
+      UPDATE_BOT_CALLBACK_QUERY = 0xb9cfc48d
+      UPDATE_CHANNEL_PARTICIPANT = 0x985d3abb
+      UPDATE_BOT_GUEST_CHAT_QUERY = 0xcdd4093d
+
+      MESSAGE_SERVICE = 0x7a800e0a
+      ACTION_CHAT_ADD_USER = 0x15cefd00
+      ACTION_JOINED_BY_LINK = 0x031224c3
+      ACTION_CHAT_DELETE_USER = 0xa43f30cc
+
+      CHANNEL_PARTICIPANT = 0x1bd54456
+      # channelParticipant / Self / Admin / Creator — a still-present member.
+      MEMBER_PARTICIPANTS = [0x1bd54456, 0xa9478a1a, 0x34c3bb53, 0x2fe601d3].freeze
+
+      REACTION_EMOJI = 0x1b2286b8
+      REACTION_EMPTY = 0x79f5d419
+      REACTION_CUSTOM_EMOJI = 0x8935fc73
+      REACTION_PAID = 0x523da4eb
+      VECTOR = 0x1cb5c415
+
+      # One Vector<Update> entry at `offset` → its Event, or nil for an update we
+      # don't surface (the caller's Schema#skip still advances past it).
+      def classify_update(data, offset, ctor)
+        case ctor
+        when UPDATE_NEW_MESSAGE, UPDATE_NEW_CHANNEL_MESSAGE
+          new_message(data, offset, channel: ctor == UPDATE_NEW_CHANNEL_MESSAGE)
+        when UPDATE_EDIT_MESSAGE, UPDATE_EDIT_CHANNEL_MESSAGE
+          edit_message(data, offset, channel: ctor == UPDATE_EDIT_CHANNEL_MESSAGE)
+        when UPDATE_MESSAGE_REACTIONS
+          reaction_event(data, offset)
+        when UPDATE_BOT_MESSAGE_REACTION
+          bot_reaction_event(data, offset)
+        when UPDATE_DELETE_CHANNEL_MESSAGES
+          delete_event(data, offset)
+        when UPDATE_BOT_CALLBACK_QUERY
+          callback_event(data, offset)
+        when UPDATE_CHANNEL_PARTICIPANT
+          participant_event(data, offset)
+        when UPDATE_BOT_GUEST_CHAT_QUERY
+          guest_query_event(data, offset)
+        end
+      end
+
+      # A Vector<Message> entry (a difference's new_messages) → a :message / :join /
+      # :leave Event WITHOUT pts (the protocol omits per-message pts there), or nil.
+      # `channel` is the envelope's kind (true for a channelDifference), since the
+      # message itself doesn't say.
+      def message_or_service(data, offset, ctor, channel:)
+        if ctor == TL::Constructors::MESSAGE
+          message_event(data, offset, ctor, kind: :message, channel: channel, pts: nil, pts_count: nil)
+        elsif ctor == MESSAGE_SERVICE
+          service_event(data, offset, channel: channel, pts: nil, pts_count: nil)
+        end
+      end
+
+      # updateNew(Channel)Message: the update tag, an inner Message (plain message, or
+      # a messageService that may be a join/leave), then pts:int pts_count:int.
+      def new_message(data, offset, channel:)
+        inner_off = offset + 4
+        inner_ctor = data[inner_off, 4].unpack1('L<')
+        pts, pts_count = pts_after(data, inner_off)
+
+        if inner_ctor == TL::Constructors::MESSAGE
+          message_event(data, inner_off, inner_ctor, kind: :message, channel: channel, pts: pts, pts_count: pts_count)
+        elsif inner_ctor == MESSAGE_SERVICE
+          service_event(data, inner_off, channel: channel, pts: pts, pts_count: pts_count)
+        end
+      end
+
+      def edit_message(data, offset, channel:)
+        inner_off = offset + 4
+        inner_ctor = data[inner_off, 4].unpack1('L<')
+        return unless inner_ctor == TL::Constructors::MESSAGE
+
+        pts, pts_count = pts_after(data, inner_off)
+        message_event(data, inner_off, inner_ctor, kind: :edit, channel: channel, pts: pts, pts_count: pts_count)
+      end
+
+      def message_event(data, offset, constructor, kind:, channel:, pts:, pts_count:)
+        msg = TL::Message.deserialize(data, offset, constructor)
+        return unless msg
+
+        Event.new(kind: kind, channel: channel, pts: pts, pts_count: pts_count,
+          peer_type: msg.peer_type, peer_id: msg.peer_id, msg_id: msg.id, message: msg.to_h)
+      end
+
+      # pts:int pts_count:int sit right after the inner Message; skip it to read them.
+      def pts_after(data, msg_off)
+        after = TL::Reader.schema.skip(data, msg_off)
+        [data[after, 4].unpack1('l<'), data[after + 4, 4].unpack1('l<')]
+      end
+
+      # messageService carrying a membership action → a :join / :leave Event, else nil.
+      # peer_id is the service message's own chat (where the event happened).
+      def service_event(data, offset, channel:, pts:, pts_count:)
+        io = offset + 4
+        flags = data[io, 4].unpack1('L<')
+        io += 8 # flags + id
+        _from_type, from_id, io = TL::Reader.parse_peer(data, io) if flags.anybits?(1 << 8) # from_id
+        peer_type, peer_id, io = TL::Reader.parse_peer(data, io)
+        io = TL::Reader.schema.skip(data, io) if flags.anybits?(1 << 28) # saved_peer_id
+        io = TL::Reader.schema.skip(data, io) if flags.anybits?(1 << 3)  # reply_to
+        io += 4 # date
+        action = data[io, 4].unpack1('L<')
+        id = data[offset + 8, 4].unpack1('l<')
+
+        event = membership_event(data, io, action, from_id, peer_type, peer_id, id)
+        return unless event
+
+        event.channel = channel
+        event.pts = pts
+        event.pts_count = pts_count
+        event
+      rescue StandardError
+        nil
+      end
+
+      def membership_event(data, io, action, from_id, peer_type, peer_id, id)
+        case action
+        when ACTION_CHAT_ADD_USER
+          joined = from_id
+          if data[io + 4, 4].unpack1('L<') == VECTOR && data[io + 8, 4].unpack1('L<').positive?
+            joined = data[io + 12, 8].unpack1('Q<')
+          end
+          Event.new(kind: :join, joined_user_id: joined, peer_type: peer_type, peer_id: peer_id, msg_id: id)
+        when ACTION_JOINED_BY_LINK
+          Event.new(kind: :join, joined_user_id: from_id, peer_type: peer_type, peer_id: peer_id, msg_id: id)
+        when ACTION_CHAT_DELETE_USER
+          Event.new(kind: :leave, left_user_id: data[io + 4, 8].unpack1('Q<'),
+            peer_type: peer_type, peer_id: peer_id, msg_id: id)
+        end
+      end
+
+      # updateChannelParticipant#985d3abb: a member joining a supergroup arrives here
+      # (not as a service message). Surface only a real join — a new_participant that
+      # is present AND was not present before.
+      def participant_event(data, offset)
+        io = offset + 4
+        flags = data[io, 4].unpack1('L<')
+        channel_id = data[io + 4, 8].unpack1('Q<')
+        user_id = data[io + 24, 8].unpack1('Q<')
+        io += 32
+
+        return unless flags.anybits?(1 << 1) # no new_participant → not a join
+
+        if flags.anybits?(1 << 0) # prev_participant
+          return if MEMBER_PARTICIPANTS.include?(data[io, 4].unpack1('L<'))
+
+          io = TL::Reader.schema.skip(data, io)
+        end
+        return unless data[io, 4].unpack1('L<') == CHANNEL_PARTICIPANT
+
+        Event.new(kind: :join, channel: true, joined_user_id: user_id, peer_type: :channel, peer_id: channel_id)
+      rescue StandardError
+        nil
+      end
+
+      # updateDeleteChannelMessages#c32d5b12 channel_id:long messages:Vector<int>
+      # pts:int pts_count:int.
+      def delete_event(data, offset)
+        io = offset + 4
+        channel_id = data[io, 8].unpack1('Q<')
+        io += 8
+        deleted_ids, io = read_int_vector(data, io)
+        Event.new(kind: :delete, channel: true, peer_type: :channel, peer_id: channel_id,
+          deleted_ids: deleted_ids, pts: data[io, 4].unpack1('l<'), pts_count: data[io + 4, 4].unpack1('l<'))
+      end
+
+      def callback_event(data, offset)
+        query = TL::Updates.extract_callback_query(data, offset)
+        Event.new(kind: :callback, query_id: query[:query_id], user_id: query[:user_id],
+          peer_type: query[:peer_type], peer_id: query[:peer_id], msg_id: query[:msg_id], data: query[:data])
+      end
+
+      def guest_query_event(data, offset)
+        query = TL::Updates.extract_guest_query(data, offset)
+        message = query[:message]
+        Event.new(kind: :guest_query, query_id: query[:query_id], qts: query[:qts], message: message,
+          peer_type: message && message[:peer_type], peer_id: message && message[:peer_id],
+          user_id: message && message[:from_id])
+      end
+
+      # updateMessageReactions# → { peer, msg_id } (the message whose reactions changed;
+      # the recent reactors are read from the message tail elsewhere).
+      def reaction_event(data, offset)
+        io = offset + 8 # constructor + flags
+        peer_type, peer_id, io = TL::Reader.parse_peer(data, io)
+        Event.new(kind: :reaction, peer_type: peer_type, peer_id: peer_id, msg_id: data[io, 4].unpack1('l<'))
+      end
+
+      # updateBotMessageReaction#ac21d3ce peer:Peer msg_id:int date:int actor:Peer
+      # old_reactions:Vector<Reaction> new_reactions:Vector<Reaction> qts:int — the
+      # bot-facing per-user reaction, the one that names the reactor. Diff old vs new to
+      # classify add (:reaction) vs remove (:unreaction).
+      def bot_reaction_event(data, offset)
+        io = offset + 4
+        peer_type, peer_id, io = TL::Reader.parse_peer(data, io)
+        msg_id = data[io, 4].unpack1('l<')
+        io += 4
+        date = data[io, 4].unpack1('L<')
+        io += 4
+        actor_type, actor_id, io = TL::Reader.parse_peer(data, io)
+        old_reactions, io = read_reactions_vector(data, io)
+        new_reactions, io = read_reactions_vector(data, io)
+        qts = data[io, 4].unpack1('l<')
+
+        added = new_reactions.reject { |r| old_reactions.any? { |o| o[:key] == r[:key] } }
+        removed = old_reactions.reject { |r| new_reactions.any? { |o| o[:key] == r[:key] } }
+        kind, diff = if added.any? then [:reaction, added]
+                     elsif removed.any? then [:unreaction, removed]
+                     end
+        return unless kind
+
+        reactor = { peer_type: actor_type, peer_id: actor_id, date: date, emoji: diff.first[:emoji] }
+        Event.new(kind: kind, msg_id: msg_id, qts: qts, peer_type: peer_type, peer_id: peer_id,
+          user_id: actor_id, reactors: [reactor])
+      rescue StandardError
+        nil
+      end
+
+      # Vector<int> at `offset` → [[int, …], offset_past].
+      def read_int_vector(data, offset)
+        offset += 4 # vector constructor
+        count = data[offset, 4].unpack1('L<')
+        offset += 4
+        ids = Array.new(count) { |i| data[offset + (i * 4), 4].unpack1('l<') }
+        [ids, offset + (count * 4)]
+      end
+
+      # Vector<Reaction> at `offset` → [ [{ key:, emoji: }, …], offset_past ]. key is a
+      # stable identity for the add/remove diff; emoji is set for plain reactions.
+      def read_reactions_vector(data, offset)
+        offset += 4 # vector constructor
+        count = data[offset, 4].unpack1('L<')
+        offset += 4
+        items = []
+        count.times do
+          key, emoji, offset = read_reaction(data, offset)
+          items << { key: key, emoji: emoji }
+        end
+        [items, offset]
+      end
+
+      # One Reaction at `offset` → [key, emoji_or_nil, offset_past]. An unknown reaction
+      # constructor is stepped over via the schema so the vector stays aligned.
+      def read_reaction(data, offset)
+        case data[offset, 4].unpack1('L<')
+        when REACTION_EMOJI
+          emoji, io = TL::Reader.read_tl_string(data, offset + 4)
+          [emoji, emoji, io]
+        when REACTION_CUSTOM_EMOJI
+          ["custom:#{data[offset + 4, 8].unpack1('Q<')}", nil, offset + 12]
+        when REACTION_PAID
+          ['paid', nil, offset + 4]
+        when REACTION_EMPTY
+          ['empty', nil, offset + 4]
+        else
+          [nil, nil, TL::Reader.schema.skip(data, offset)]
+        end
+      end
+    end
+  end
+end

data/lib/mtproto/updates/memory_store.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module MTProto
+  module Updates
+    # In-memory update-state store: the default backend behind the state machine's
+    # persistence port. A real host swaps in its own object answering the same
+    # messages over DURABLE storage — load_state / save_state, channel_pts /
+    # set_channel_pts, channel_access / set_channel_access, transaction.
+    #
+    # load_state returns { pts:, qts:, date:, seq: } or nil on a cold start.
+    # A channel's access_hash is persisted next to its pts because a bot cannot
+    # getDialogs — it learns the hash only by seeing the channel — so without it a gap
+    # in a channel not seen since the last restart could not be recovered.
+    # transaction yields; a host may wrap it — the block runs event delivery plus the
+    # pts checkpoint — in one DB transaction, so a crash mid-handler re-delivers the
+    # update instead of skipping it.
+    class MemoryStore
+      def initialize
+        @state = nil
+        @channel_pts = {}
+        @channel_access = {}
+      end
+
+      def load_state
+        @state&.dup
+      end
+
+      def save_state(pts:, qts:, date:, seq:)
+        @state = { pts: pts, qts: qts, date: date, seq: seq }
+      end
+
+      def channel_pts(channel_id)
+        @channel_pts[channel_id]
+      end
+
+      def set_channel_pts(channel_id, pts)
+        @channel_pts[channel_id] = pts
+      end
+
+      def channel_access(channel_id)
+        @channel_access[channel_id]
+      end
+
+      def set_channel_access(channel_id, access_hash)
+        @channel_access[channel_id] = access_hash
+      end
+
+      def transaction
+        yield
+      end
+    end
+  end
+end

data/lib/mtproto/updates/parser.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+
+require_relative '../tl/reader'
+require_relative '../tl/objects/dialogs'
+require_relative '../tl/objects/updates_state'
+require_relative 'event_builder'
+
+module MTProto
+  module Updates
+    # Parses a raw Telegram update reply into normalized Events, shared by the live
+    # push path and the getDifference/getChannelDifference recovery paths so both
+    # yield one event shape. This module walks the ENVELOPE (which container, which
+    # vector, the trailing peers and state); EventBuilder turns each Update into an
+    # Event. Every vector entry is advanced via Schema#skip, so an update type we
+    # don't model never drops the ones around it.
+    module Parser
+      module_function
+
+      UPDATES = 0x74ae4240
+      UPDATES_COMBINED = 0x725b04c3
+      UPDATES_TOO_LONG = 0xe317af7e
+      UPDATE_SHORT = 0x78d4dec1
+
+      UPDATES_DIFFERENCE = 0x00f49ca0
+      UPDATES_DIFFERENCE_SLICE = 0xa8fb1981
+      UPDATES_DIFFERENCE_EMPTY = 0x5d75a138
+      UPDATES_DIFFERENCE_TOO_LONG = 0x4afe8f6d
+
+      CHANNEL_DIFFERENCE = 0x2064674e
+      CHANNEL_DIFFERENCE_EMPTY = 0x3e11affb
+      CHANNEL_DIFFERENCE_TOO_LONG = 0xa4bcc6fe
+
+      UPDATE_CHANNEL_TOO_LONG = 0x108d941f
+
+      # Batch is the live-container result; Difference is the recovery-reply result and
+      # additionally carries the new authoritative position (pts/qts/date/seq), or, for
+      # a channel, the new channel pts + `final` (whether the backlog is drained).
+      Batch = Struct.new(:events, :users, :chats, :channel_too_long, :too_long, :date, keyword_init: true)
+      Difference = Struct.new(:type, :final, :events, :users, :chats, :channel_too_long,
+        :pts, :qts, :date, :seq, keyword_init: true)
+
+      # A live `updates` / `updatesCombined` push container (or a bodyless
+      # `updatesTooLong` gap signal) → a Batch of ordered events plus the batch peers
+      # and gap signals.
+      def parse_container(constructor, data)
+        case constructor
+        when UPDATES_TOO_LONG
+          Batch.new(events: [], users: [], chats: [], channel_too_long: [], too_long: true, date: nil)
+        when UPDATES, UPDATES_COMBINED
+          parse_updates(data)
+        when UPDATE_SHORT
+          parse_short(data)
+        else
+          Batch.new(events: [], users: [], chats: [], channel_too_long: [], too_long: false, date: nil)
+        end
+      end
+
+      # updateShort#78d4dec1 update:Update date:int — a single update with no peers
+      # attached. Channel messages never arrive this way, but edits/reactions do, so
+      # classify the one update (or surface an updateChannelTooLong gap signal).
+      def parse_short(data)
+        inner_ctor = data[4, 4].unpack1('L<')
+        if inner_ctor == UPDATE_CHANNEL_TOO_LONG
+          return Batch.new(events: [], users: [], chats: [], channel_too_long: [data[12, 8].unpack1('Q<')],
+            too_long: false, date: nil)
+        end
+
+        event = EventBuilder.classify_update(data, 4, inner_ctor)
+        Batch.new(events: [event].compact, users: [], chats: [], channel_too_long: [],
+          too_long: false, date: data[TL::Reader.schema.skip(data, 4), 4].unpack1('L<'))
+      end
+
+      # updates: updates:Vector<Update> users:Vector<User> chats:Vector<Chat> date:int
+      # (updatesCombined only inserts seq_start before the trailing seq, past date).
+      def parse_updates(data)
+        events, channel_too_long, offset = updates_vector(data, 4)
+        users, offset = TL::Dialogs.users_at(data, offset)
+        chats, offset = TL::Dialogs.chats_at(data, offset)
+        date = data[offset, 4].unpack1('L<')
+        attach_peers(events, users, chats)
+        Batch.new(events: events, users: users, chats: chats,
+          channel_too_long: channel_too_long, too_long: false, date: date)
+      end
+
+      # updates.difference / differenceSlice reply → a Difference with the recovered
+      # events and the new account position (in the trailing updates.State). Common
+      # new_messages arrive WITHOUT a per-message pts (the protocol omits it), so their
+      # events carry pts nil; the state machine advances to the batch-terminal state.pts.
+      def parse_difference(data)
+        case data[0, 4].unpack1('L<')
+        when UPDATES_DIFFERENCE_EMPTY
+          empty_difference(date: data[4, 4].unpack1('L<'), seq: data[8, 4].unpack1('L<'))
+        when UPDATES_DIFFERENCE, UPDATES_DIFFERENCE_SLICE
+          difference_body(data, data[0, 4].unpack1('L<'))
+        when UPDATES_DIFFERENCE_TOO_LONG
+          empty_difference(type: :too_long, pts: data[4, 4].unpack1('l<'))
+        else
+          empty_difference
+        end
+      end
+
+      # updates.channelDifference reply → a Difference carrying the channel's new pts
+      # and `final`. channel messages ride in new_messages without a per-message pts;
+      # the state machine advances the channel pts to this reply's pts.
+      def parse_channel_difference(data)
+        case data[0, 4].unpack1('L<')
+        when CHANNEL_DIFFERENCE_EMPTY
+          flags = data[4, 4].unpack1('L<')
+          empty_difference(final: flags.anybits?(1 << 0), pts: data[8, 4].unpack1('l<'))
+        when CHANNEL_DIFFERENCE
+          channel_difference_body(data)
+        when CHANNEL_DIFFERENCE_TOO_LONG
+          empty_difference(type: :too_long, final: true)
+        else
+          empty_difference(final: true)
+        end
+      end
+
+      def empty_difference(type: :empty, final: false, pts: nil, date: nil, seq: nil)
+        Difference.new(type: type, final: final, events: [], users: [], chats: [],
+          channel_too_long: [], pts: pts, date: date, seq: seq)
+      end
+
+      # difference / differenceSlice: new_messages new_encrypted_messages other_updates
+      # chats users state. Channel messages ride in other_updates (with pts); common
+      # new_messages ride in new_messages (no pts).
+      def difference_body(data, constructor)
+        msg_events, offset = messages_vector(data, 4, channel: false)
+        offset = TL::Reader.schema.skip_vector(data, offset) # new_encrypted_messages
+        upd_events, channel_too_long, offset = updates_vector(data, offset)
+        chats, offset = TL::Dialogs.chats_at(data, offset)
+        users, offset = TL::Dialogs.users_at(data, offset)
+        state = TL::UpdatesState.deserialize(data[offset..])
+
+        events = msg_events + upd_events
+        attach_peers(events, users, chats)
+        Difference.new(
+          type: constructor == UPDATES_DIFFERENCE ? :difference : :slice, final: false,
+          events: events, users: users, chats: chats, channel_too_long: channel_too_long,
+          pts: state.pts, qts: state.qts, date: state.date, seq: state.seq
+        )
+      end
+
+      # channelDifference#2064674e flags:# final:flags.0?true pts:int timeout:flags.1?int
+      #   new_messages:Vector<Message> other_updates:Vector<Update> chats users.
+      def channel_difference_body(data)
+        flags = data[4, 4].unpack1('L<')
+        pts = data[8, 4].unpack1('l<')
+        offset = 12
+        offset += 4 if flags.anybits?(1 << 1) # timeout
+
+        msg_events, offset = messages_vector(data, offset, channel: true)
+        upd_events, channel_too_long, offset = updates_vector(data, offset)
+        chats, offset = TL::Dialogs.chats_at(data, offset)
+        users, = TL::Dialogs.users_at(data, offset)
+
+        events = msg_events + upd_events
+        attach_peers(events, users, chats)
+        Difference.new(type: :difference, final: flags.anybits?(1 << 0), pts: pts,
+          events: events, users: users, chats: chats, channel_too_long: channel_too_long)
+      end
+
+      # A Vector<Message> (a difference's new_messages) → ordered :message / :join /
+      # :leave events WITHOUT pts (the protocol omits per-message pts here).
+      def messages_vector(data, offset, channel:)
+        offset += 4 # vector constructor
+        count = data[offset, 4].unpack1('L<')
+        offset += 4
+
+        events = []
+        count.times do
+          event = EventBuilder.message_or_service(data, offset, data[offset, 4].unpack1('L<'), channel: channel)
+          events << event if event
+          offset = TL::Reader.schema.skip(data, offset)
+        end
+        [events, offset]
+      end
+
+      # A Vector<Update> (a live container's updates, or a difference's other_updates)
+      # → [ordered events, updateChannelTooLong channel ids, offset past the vector].
+      def updates_vector(data, offset)
+        offset += 4 # vector constructor
+        count = data[offset, 4].unpack1('L<')
+        offset += 4
+
+        events = []
+        channel_too_long = []
+        count.times do
+          ctor = data[offset, 4].unpack1('L<')
+          if ctor == UPDATE_CHANNEL_TOO_LONG
+            channel_too_long << data[offset + 8, 8].unpack1('Q<') # constructor(4) + flags(4)
+          else
+            event = EventBuilder.classify_update(data, offset, ctor)
+            events << event if event
+          end
+          offset = TL::Reader.schema.skip(data, offset)
+        end
+        [events, channel_too_long, offset]
+      end
+
+      def attach_peers(events, users, chats)
+        events.each do |event|
+          event.users = users
+          event.chats = chats
+        end
+      end
+    end
+  end
+end

data/lib/mtproto/updates/runner.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'async/queue'
+require_relative 'state_machine'
+
+module MTProto
+  module Updates
+    # Drives the high-level update loop over a Client (Client#run_updates is the entry
+    # point). The on_update callback runs on the receiver fiber and only ENQUEUES raw
+    # containers; a single worker fiber drains the queue and feeds the state machine,
+    # so recovery RPCs never block the receiver (no deadlock) and never race each other
+    # on pts (serial by construction). A reconnect enqueues a re-sync. An optional
+    # on_start runs once inside the started mainloop before the first update, the point
+    # at which host startup that needs the API belongs.
+    class Runner
+      def initialize(client:, store:, logger: nil)
+        @client = client
+        @store = store
+        @logger = logger
+        @machine = StateMachine.new(client: client, store: store, logger: logger)
+      end
+
+      def run(on_start: nil, &block)
+        raise ArgumentError, 'block is required' unless block
+
+        @client.run_mainloop do
+          queue = Async::Queue.new
+          @client.replace_update_callback { |constructor, body| queue.enqueue([constructor, body]) }
+          @client.on_reconnect { queue.enqueue(:reconnect) }
+          on_start&.call
+          @machine.resume(&block)
+          drain(queue, &block)
+        end
+      end
+
+      private
+
+      # A handler error is logged and swallowed so one bad update never kills the loop.
+      def drain(queue, &)
+        loop do
+          item = queue.dequeue
+          if item == :reconnect
+            @machine.resync(&)
+          else
+            @machine.process(item[0], item[1], &)
+          end
+        rescue StandardError => e
+          @logger&.error("mtproto updates: #{e.class}: #{e.message}")
+        end
+      end
+    end
+  end
+end

data/lib/mtproto/updates/state_machine.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require_relative 'parser'
+
+module MTProto
+  module Updates
+    # The authoritative update state machine: owns the account position
+    # (pts/qts/date/seq) and per-channel pts, applies live updates in order, and
+    # recovers gaps via getDifference/getChannelDifference — but ONLY when a real
+    # sequence gap is detected, never on a blind timer. Because a gap-triggered
+    # getDifference returns exactly the missed range and the position is advanced
+    # atomically, the host never sees a duplicate in steady state (a crash
+    # mid-handler may re-deliver one update — the accepted at-least-once edge).
+    #
+    # Sequential and Async-free: it issues blocking RPCs through the injected client,
+    # so it MUST be driven off the receiver fiber (run_updates feeds it from a single
+    # worker fiber, which also serializes recovery). Checkpoints run through
+    # store.transaction, so a host can commit the pts advance together with its own
+    # handling of the event.
+    class StateMachine
+      RECOVERY_PAGES = 500
+
+      def initialize(client:, store:, logger: nil)
+        @client = client
+        @store = store
+        @logger = logger
+      end
+
+      def resume(&)
+        stored = @store.load_state
+        base = @client.api.get_updates_state
+        @pts = nonzero(stored && stored[:pts]) || base.pts
+        @qts = nonzero(stored && stored[:qts]) || base.qts
+        @date = nonzero(stored && stored[:date]) || base.date
+        @seq = nonzero(stored && stored[:seq]) || base.seq
+        persist_state
+        catch_up(&) if stored
+      end
+
+      # Feed one raw live update container (constructor + body) from the push stream.
+      def process(constructor, body, &)
+        dispatch(Parser.parse_container(constructor, body), &)
+      end
+
+      # A gap point the host detected out of band (notably a reconnect): re-sync now.
+      def resync(&)
+        catch_up(&)
+      end
+
+      private
+
+      def dispatch(batch, &)
+        remember_channels(batch.chats)
+        batch.events.each { |event| ingest(event, &) }
+        batch.channel_too_long.each { |channel_id| recover_channel(channel_id, &) }
+        catch_up(&) if batch.too_long
+      end
+
+      def ingest(event, &)
+        detect_gap(event, &)
+        emit(event, &)
+      end
+
+      # A live update whose pts jumps past our stored position means updates were
+      # dropped in between; pull the hole before applying the update that revealed it.
+      # A channel needs a known local baseline first (its first update seeds it).
+      def detect_gap(event, &)
+        return unless event.pts && event.pts_count
+
+        if event.channel
+          local = @store.channel_pts(event.peer_id)
+          recover_channel(event.peer_id, &) if local && (event.pts - event.pts_count) > local
+        elsif @pts && (event.pts - event.pts_count) > @pts
+          catch_up(&)
+        end
+      end
+
+      # A re-delivery we already passed (pts <= local) is dropped; otherwise deliver the
+      # event and persist the position it advances. The store write runs inside the
+      # host's transaction (atomic with the host's own handling); the in-memory @pts is
+      # advanced only AFTER the transaction commits, so a rollback can never leave the
+      # cache ahead of the store and silently drop later account updates.
+      def emit(event, &block)
+        return if outdated?(event)
+
+        @store.transaction do
+          block.call(event)
+          checkpoint(event)
+        end
+        cache_account_position(event)
+      end
+
+      def outdated?(event)
+        return false unless event.pts
+
+        if event.channel
+          local = @store.channel_pts(event.peer_id)
+          local && event.pts <= local
+        else
+          @pts && event.pts <= @pts
+        end
+      end
+
+      def checkpoint(event)
+        return unless event.pts
+
+        if event.channel
+          local = @store.channel_pts(event.peer_id)
+          @store.set_channel_pts(event.peer_id, event.pts) if local.nil? || event.pts > local
+        elsif account_newer?(event)
+          @store.save_state(pts: event.pts, qts: @qts, date: @date, seq: @seq)
+        end
+      end
+
+      def cache_account_position(event)
+        @pts = event.pts if !event.channel && event.pts && account_newer?(event)
+      end
+
+      def account_newer?(event)
+        @pts.nil? || event.pts > @pts
+      end
+
+      def catch_up(&)
+        RECOVERY_PAGES.times do
+          diff = fetch_difference
+          case diff.type
+          when :empty
+            @date = diff.date if diff.date
+            @seq = diff.seq if diff.seq
+            persist_state
+            return
+          when :difference, :slice
+            apply_difference(diff, &)
+            return if diff.type == :difference # a slice means more remains; keep pulling
+          when :too_long
+            resync_to_current
+            return
+          end
+        end
+        @logger&.warn('mtproto updates: getDifference catch-up hit the page cap')
+      end
+
+      def apply_difference(diff, &)
+        remember_channels(diff.chats)
+        diff.events.each { |event| emit(event, &) }
+        diff.channel_too_long.each { |channel_id| recover_channel(channel_id, &) }
+        @pts = diff.pts if diff.pts
+        @qts = diff.qts if diff.qts
+        @date = diff.date if diff.date
+        @seq = diff.seq if diff.seq
+        persist_state
+      end
+
+      # Recover one channel via getChannelDifference from its stored pts. Without a
+      # known access_hash or local pts we can't target it, so fall back to the
+      # account-wide getDifference (whose reply carries the channel's messages too).
+      def recover_channel(channel_id, &)
+        access = @store.channel_access(channel_id)
+        from = @store.channel_pts(channel_id)
+        return catch_up(&) unless access && from
+
+        RECOVERY_PAGES.times do
+          diff = fetch_channel_difference(channel_id, access, from)
+          return catch_up(&) if diff.type == :too_long
+
+          remember_channels(diff.chats)
+          diff.events.each { |event| emit(event, &) }
+          if diff.pts&.positive?
+            from = diff.pts
+            @store.set_channel_pts(channel_id, diff.pts)
+          end
+          return if diff.final
+        end
+        @logger&.warn("mtproto updates: channel #{channel_id} recovery hit the page cap")
+      end
+
+      def resync_to_current
+        base = @client.api.get_updates_state
+        @pts = base.pts
+        @qts = base.qts
+        @date = base.date
+        @seq = base.seq
+        persist_state
+      end
+
+      def fetch_difference
+        Parser.parse_difference(@client.api.get_updates_difference_raw(pts: @pts, date: @date, qts: @qts))
+      end
+
+      def fetch_channel_difference(channel_id, access_hash, pts)
+        Parser.parse_channel_difference(
+          @client.api.get_channel_difference_raw(channel: { id: channel_id, access_hash: access_hash }, pts: pts,
+            limit: 100)
+        )
+      end
+
+      # A bot learns a channel's access_hash only by seeing it; persist the latest so a
+      # gap — even after a restart, before the channel is seen again — can be recovered
+      # with a targeted getChannelDifference rather than a lossy account-wide fallback.
+      def remember_channels(chats)
+        Array(chats).each do |chat|
+          @store.set_channel_access(chat.id, chat.access_hash) if chat.access_hash
+        end
+      end
+
+      def persist_state
+        @store.save_state(pts: @pts, qts: @qts, date: @date, seq: @seq)
+      end
+
+      def nonzero(value)
+        value if value&.nonzero?
+      end
+    end
+  end
+end

data/lib/mtproto/updates.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+require_relative 'updates/event'
+require_relative 'updates/event_builder'
+require_relative 'updates/parser'
+require_relative 'updates/memory_store'
+require_relative 'updates/state_machine'
+require_relative 'updates/runner'

data/lib/mtproto/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module MTProto
-  VERSION = '0.0.27'
+  VERSION = '0.0.29'
 end

data/lib/mtproto.rb

@@ -37,6 +37,7 @@ require_relative 'mtproto/crypto/message_key'
 require_relative 'mtproto/auth_key_generator'
 require_relative 'mtproto/session'
 require_relative 'mtproto/encrypted_message'
+require_relative 'mtproto/updates'
 require_relative 'mtproto/client'
 require_relative 'mtproto/file_downloader'
 require_relative 'mtproto/bot_authorizer'

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mtproto
 version: !ruby/object:Gem::Version
-  version: 0.0.27
+  version: 0.0.29
 platform: ruby
 authors:
 - Artem Levenkov
@@ -86,6 +86,7 @@ files:
 - lib/mtproto/client/api/unpin_all_messages.rb
 - lib/mtproto/client/rpc.rb
 - lib/mtproto/client/rpc/response.rb
+- lib/mtproto/client/update_api.rb
 - lib/mtproto/crypto/aes_ige.rb
 - lib/mtproto/crypto/auth_key_helper.rb
 - lib/mtproto/crypto/dh_key_exchange.rb
@@ -225,6 +226,13 @@ files:
 - lib/mtproto/transport/packet.rb
 - lib/mtproto/transport/tcp_connection.rb
 - lib/mtproto/unencrypted_message.rb
+- lib/mtproto/updates.rb
+- lib/mtproto/updates/event.rb
+- lib/mtproto/updates/event_builder.rb
+- lib/mtproto/updates/memory_store.rb
+- lib/mtproto/updates/parser.rb
+- lib/mtproto/updates/runner.rb
+- lib/mtproto/updates/state_machine.rb
 - lib/mtproto/version.rb
 - scripts/gen_constructor_names.rb
 - scripts/generate_constructors.rb