mtproto 0.0.19 → 0.0.20

data/lib/mtproto/markdown.rb

@@ -0,0 +1,439 @@
+# frozen_string_literal: true
+
+require_relative 'tl/message_entity'
+
+module MTProto
+  # Telegram-MarkdownV2-style formatting: text markup <-> (plain text, entities).
+  #
+  #   text, entities = MTProto::Markdown.parse("*bold* and _italic_ and ||spoiler||")
+  #   markup         = MTProto::Markdown.render(text, entities)
+  #
+  # Entity offsets/lengths are in **UTF-16 code units** (Telegram's unit), so the
+  # parser counts astral characters (emoji) as 2. Supported markup:
+  #   *bold*  _italic_  __underline__  ~strikethrough~  ||spoiler||
+  #   `inline code`   ```lang\npre block```
+  #   [label](http://url)              -> text_url
+  #   [label](tg://user?id=123)        -> mention (text-mention)
+  #   ![emoji](tg://emoji?id=123)      -> custom emoji
+  #   >quoted line(s)                  -> blockquote
+  #   \x escapes any markup character x.
+  #
+  # Malformed markup (an unclosed entity, a broken link) raises ParseError, matching
+  # Telegram's own "can't parse entities" rejection rather than sending garbage.
+  #
+  # Known limitations (vs full Telegram MarkdownV2): expandable/collapsed blockquotes
+  # are not produced; adjacent italic/underline runs (the `_`/`__` adjacency, e.g.
+  # `___x___`) are rejected as ambiguous; link labels are literal text (no nested
+  # formatting inside `[...]`); auto-detected entities (url/mention/hashtag/email/…)
+  # are left to Telegram's server-side detection and not emitted by parse.
+  module Markdown
+    class ParseError < StandardError; end
+
+    Entity = MTProto::TL::MessageEntity
+
+    module_function
+
+    def parse(source)
+      Parser.new(utf8(source)).parse
+    end
+
+    def render(text, entities)
+      Renderer.new(utf8(text), entities).render
+    end
+
+    def utf16_size(char)
+      char.ord > 0xFFFF ? 2 : 1
+    end
+
+    # Treat input as UTF-8 text and reject invalid bytes with a clear error rather
+    # than letting a low-level ArgumentError/Encoding error escape from deep inside.
+    def utf8(str)
+      s = str.to_s
+      s = s.dup.force_encoding(Encoding::UTF_8) unless s.encoding == Encoding::UTF_8
+      raise ParseError, 'input is not valid UTF-8' unless s.valid_encoding?
+
+      s
+    end
+
+    # Single-pass scanner. Paired markers (bold/italic/underline/strike/spoiler) use
+    # a nesting stack; code/pre/links/custom-emoji are consumed as self-contained
+    # spans; blockquotes are recognised at line starts. Everything is tracked in
+    # UTF-16 units so the emitted entity offsets match the wire.
+    class Parser
+      MARKERS = { '||' => :spoiler, '__' => :underline, '*' => :bold, '_' => :italic, '~' => :strike }.freeze
+      # Longest first so `__`/`||` win over `_`/`|`.
+      MARKER_ORDER = ['```', '||', '__', '`', '*', '_', '~'].freeze
+
+      def initialize(source)
+        @chars = source.to_s.chars
+        @len = @chars.length
+        @i = 0
+        @text = +''
+        @u16 = 0
+        @stack = []
+        @entities = []
+        @quote = nil
+        @at_line_start = true
+      end
+
+      def parse
+        scan while @i < @len
+        close_quote
+        raise ParseError, "unclosed #{@stack.last[:type]}" unless @stack.empty?
+
+        # Drop zero-length entities (e.g. from empty markup like "**") — Telegram
+        # rejects them with ENTITY_BOUNDS_INVALID.
+        kept = @entities.reject { |e| e.length.to_i <= 0 }
+        [@text, kept.sort_by { |e| [e.offset, -e.length] }]
+      end
+
+      private
+
+      def scan
+        handle_line_start if @at_line_start
+        return if @i >= @len # a bare '>' line at EOF advances past the end
+
+        c = @chars[@i]
+
+        return escape if c == '\\'
+        return newline if c == "\n"
+        return if try_custom_emoji || try_link || try_marker
+
+        emit(c)
+        @i += 1
+      end
+
+      # A blockquote is a run of lines each starting with '>'. Reaching a line that
+      # doesn't (or EOF) closes it. code/pre spans are consumed whole (incl. their
+      # newlines) by open_code/open_pre, so a line start is never reached inside one.
+      def handle_line_start
+        @at_line_start = false
+
+        if @chars[@i] == '>'
+          open_quote
+          @i += 1
+          @i += 1 if @chars[@i] == ' ' # one optional space after '>'
+        else
+          close_quote
+        end
+      end
+
+      def open_quote
+        return if @quote
+
+        @quote = { start: @u16, end: @u16 }
+      end
+
+      # Emit the blockquote entity for the accumulated quoted lines, ending at the
+      # last quoted content char (a trailing newline before a non-quote line is not
+      # part of the quote). Expandable/collapsed quotes are not produced (the `||`
+      # end-mark collides with spoiler markup), so `collapsed` is always false.
+      def close_quote
+        return unless @quote
+
+        @entities << Entity.new(type: :blockquote, offset: @quote[:start],
+          length: @quote[:end] - @quote[:start], collapsed: false)
+        @quote = nil
+      end
+
+      def newline
+        emit("\n")
+        @i += 1
+        @at_line_start = true
+      end
+
+      def escape
+        nxt = @chars[@i + 1]
+        raise ParseError, 'dangling escape at end of input' if nxt.nil?
+
+        emit(nxt)
+        @i += 2
+      end
+
+      def emit(char)
+        @text << char
+        @u16 += Markdown.utf16_size(char)
+        @quote[:end] = @u16 if @quote && char != "\n"
+      end
+
+      # ![label](tg://emoji?id=N) -> custom_emoji. Only when a real emoji id parses.
+      def try_custom_emoji
+        return false unless @chars[@i] == '!' && @chars[@i + 1] == '['
+
+        save = @i
+        @i += 1
+        unless try_link(custom_emoji: true)
+          @i = save
+          return false
+        end
+        true
+      end
+
+      # [label](url). label is literal text (escapes honoured, no nested markup);
+      # url picks the entity kind: tg://user?id / tg://emoji?id / anything else. On
+      # any mismatch @i is untouched (the reads use a local cursor), so '[' falls
+      # through to a literal.
+      def try_link(custom_emoji: false)
+        return false unless @chars[@i] == '['
+
+        label, after_label = read_link_label(@i + 1)
+        return false if label.nil?
+        return false unless @chars[after_label] == '('
+
+        url, after_url = read_link_url(after_label + 1)
+        return false if url.nil?
+
+        entity = link_entity(url, custom_emoji: custom_emoji)
+        return false if entity.nil?
+
+        entity.offset = @u16
+        label.each_char { |ch| emit(ch) }
+        entity.length = @u16 - entity.offset
+        @entities << entity
+        @i = after_url
+        true
+      end
+
+      # Returns [label, index_just_after_']'] or [nil, _] when no ']' on this line.
+      def read_link_label(from)
+        out = +''
+        j = from
+        while j < @len
+          ch = @chars[j]
+          if ch == '\\' && j + 1 < @len
+            out << @chars[j + 1]
+            j += 2
+          elsif ch == ']'
+            return [out, j + 1]
+          elsif ch == "\n"
+            return [nil, from]
+          else
+            out << ch
+            j += 1
+          end
+        end
+        [nil, from]
+      end
+
+      def read_link_url(from)
+        out = +''
+        j = from
+        while j < @len
+          ch = @chars[j]
+          if ch == '\\' && j + 1 < @len
+            out << @chars[j + 1]
+            j += 2
+          elsif ch == ')'
+            return [out, j + 1]
+          elsif ch == "\n"
+            return [nil, from]
+          else
+            out << ch
+            j += 1
+          end
+        end
+        [nil, from]
+      end
+
+      def link_entity(url, custom_emoji:)
+        if custom_emoji
+          id = url[%r{\Atg://emoji\?id=(\d+)\z}, 1]
+          return id && Entity.new(type: :custom_emoji, document_id: id.to_i)
+        end
+        if (id = url[%r{\Atg://user\?id=(\d+)\z}, 1])
+          return Entity.new(type: :mention_name, user_id: id.to_i)
+        end
+
+        Entity.new(type: :text_url, url: url)
+      end
+
+      def try_marker
+        MARKER_ORDER.each do |m|
+          next unless @chars[@i, m.length].join == m
+
+          @i += m.length
+          apply_marker(m)
+          return true
+        end
+        false
+      end
+
+      def apply_marker(marker)
+        return open_pre if marker == '```'
+        return open_code if marker == '`'
+
+        type = MARKERS.fetch(marker)
+        if @stack.last && @stack.last[:marker] == marker
+          top = @stack.pop
+          @entities << Entity.new(type: type, offset: top[:start], length: @u16 - top[:start])
+        elsif @stack.any? { |s| s[:marker] == marker }
+          raise ParseError, "improperly nested #{type}"
+        else
+          @stack << { marker: marker, type: type, start: @u16 }
+        end
+      end
+
+      # `inline code` — content is literal up to the next backtick (\` escapes one).
+      def open_code
+        start = @u16
+        while @i < @len
+          c = @chars[@i]
+          if c == '\\' && @chars[@i + 1] == '`'
+            emit('`')
+            @i += 2
+          elsif c == '`'
+            @i += 1
+            @entities << Entity.new(type: :code, offset: start, length: @u16 - start)
+            return
+          else
+            emit(c)
+            @i += 1
+          end
+        end
+        raise ParseError, 'unclosed code span'
+      end
+
+      # ```lang\ncontent``` — the first line (up to \n) is the language when it is a
+      # bare token; otherwise there is no language and the whole body is content.
+      def open_pre
+        close = find_pre_close(@i)
+        raise ParseError, 'unclosed pre block' if close.nil?
+
+        body = @chars[@i...close].join
+        language, content = split_pre_language(body)
+        start = @u16
+        content.each_char { |ch| emit(ch) }
+        @entities << Entity.new(type: :pre, offset: start, length: @u16 - start, language: language)
+        @i = close + 3
+      end
+
+      def find_pre_close(from)
+        j = from
+        while j < @len
+          return j if @chars[j, 3].join == '```'
+
+          j += 1
+        end
+        nil
+      end
+
+      def split_pre_language(body)
+        nl = body.index("\n")
+        return ['', body] if nl.nil?
+
+        first = body[0...nl]
+        return [first, body[(nl + 1)..]] if first.match?(/\A[A-Za-z0-9_+.-]*\z/)
+
+        ['', body]
+      end
+    end
+
+    # entities -> MarkdownV2 markup. Escapes literal markup characters and wraps each
+    # entity's UTF-16 range with its markers. Entities are opened outer-first and
+    # closed inner-first so proper nesting round-trips. Works in UTF-16 unit space.
+    class Renderer
+      SPECIAL = '_*[]()~`>#+-=|{}.!\\'
+      WRAP = { bold: '*', italic: '_', underline: '__', strike: '~', spoiler: '||' }.freeze
+
+      def initialize(text, entities)
+        @units = (text || '').encode('UTF-16LE').unpack('v*')
+        @entities = (entities || []).map { |e| e.is_a?(Hash) ? Entity.new(**e) : e }
+      end
+
+      def render
+        opens = Hash.new { |h, k| h[k] = [] }
+        closes = Hash.new { |h, k| h[k] = [] }
+        @entities.each_with_index do |e, i|
+          opens[e.offset] << [e, i]
+          closes[e.offset + e.length] << [e, i]
+        end
+        verbatim = ranges_of(%i[code pre]) # content here is literal — never escaped
+        quotes = ranges_of(%i[blockquote]) # rendered as a '>' line prefix
+
+        out = +''
+        pos = 0
+        line_start = true
+        loop do
+          # Close inner-first, the exact mirror of the open order (outer-first), so
+          # entities that end together don't emit crossed markers. Ties broken by the
+          # reverse of the open tiebreak.
+          closes[pos].sort_by { |(e, i)| [e.length, -i] }.each { |(e, _)| out << close_marker(e) }
+          out << '>' if line_start && in_range?(quotes, pos)
+          opens[pos].sort_by { |(e, i)| [-e.length, i] }.each { |(e, _)| out << open_marker(e) }
+          break if pos >= @units.length
+
+          u = @units[pos]
+          if high_surrogate?(u) && pos + 1 < @units.length
+            out << pair_to_s(u, @units[pos + 1]) # an astral char never needs escaping
+            pos += 2
+            line_start = false
+          else
+            ch = unit_to_s(u)
+            out << render_char(ch, in_range?(verbatim, pos))
+            line_start = ch == "\n"
+            pos += 1
+          end
+        end
+        out
+      end
+
+      private
+
+      def ranges_of(types)
+        @entities.select { |e| types.include?(e.type) }.map { |e| e.offset...(e.offset + e.length) }
+      end
+
+      def in_range?(ranges, pos)
+        ranges.any? { |r| r.cover?(pos) }
+      end
+
+      def high_surrogate?(unit)
+        unit.between?(0xD800, 0xDBFF)
+      end
+
+      def pair_to_s(high, low)
+        [high, low].pack('v2').force_encoding('UTF-16LE').encode('UTF-8')
+      end
+
+      def unit_to_s(unit)
+        [unit].pack('v').force_encoding('UTF-16LE').encode('UTF-8')
+      end
+
+      # Literal chars inside code/pre are emitted verbatim; elsewhere MarkdownV2
+      # specials are backslash-escaped so they re-parse as text, not markup.
+      def render_char(char, verbatim)
+        return char if verbatim
+
+        SPECIAL.include?(char) ? "\\#{char}" : char
+      end
+
+      def open_marker(entity)
+        case entity.type
+        when :bold, :italic, :underline, :strike, :spoiler then WRAP.fetch(entity.type)
+        when :code then '`'
+        when :pre then entity.language.to_s.empty? ? "```\n" : "```#{entity.language}\n"
+        when :text_url, :mention_name, :custom_emoji then entity.type == :custom_emoji ? '![' : '['
+        else '' # blockquote is prefix-based and not wrapped here; unknown types no-op
+        end
+      end
+
+      def close_marker(entity)
+        case entity.type
+        when :bold, :italic, :underline, :strike, :spoiler then WRAP.fetch(entity.type)
+        when :code then '`'
+        when :pre then '```'
+        when :text_url then "](#{escape_url(entity.url)})"
+        when :mention_name then "](tg://user?id=#{entity.user_id})"
+        when :custom_emoji then "](tg://emoji?id=#{entity.document_id})"
+        else ''
+        end
+      end
+
+      # Inside a link's (...) the parser stops the URL at the first ')' and treats
+      # '\' as an escape, so those two must be escaped to survive a re-parse.
+      def escape_url(url)
+        url.to_s.gsub(/[\\)]/) { |c| "\\#{c}" }
+      end
+    end
+  end
+end

data/lib/mtproto/tl/constructors.rb

@@ -13,6 +13,15 @@ module MTProto
       BAD_SERVER_SALT = 0xedab447b
       BAD_MSG_NOTIFICATION = 0xa7eff811
       NEW_SESSION_CREATED = 0x9ec20908
+      PONG = 0x347773c5
+      MSGS_ACK = 0x62d6b459
+      MSGS_STATE_REQ = 0xda69fb52
+      MSGS_STATE_INFO = 0x04deb57d
+      MSGS_ALL_INFO = 0x8cc0d131
+      MSG_DETAILED_INFO = 0x276d3ec6
+      MSG_NEW_DETAILED_INFO = 0x809db6df
+      FUTURE_SALT = 0x0949d9dc
+      FUTURE_SALTS = 0xae500895
 
       # Peers
       PEER_USER = 0x59511722
@@ -108,10 +117,13 @@ module MTProto
       # Updates
       UPDATES = 0x74ae4240
       UPDATES_COMBINED = 0x725b04c3
+      UPDATES_TOO_LONG = 0xe317af7e
       UPDATE_NEW_MESSAGE = 0x1f2b0afd
       UPDATE_SHORT = 0x78d4dec1
       UPDATE_SHORT_MESSAGE = 0x313bc7f8
+      UPDATE_SHORT_CHAT_MESSAGE = 0x4d6deea5
       UPDATE_SHORT_SENT_MESSAGE = 0x9015e101
+      UPDATE_LOGIN_TOKEN = 0x564fe691
 
       # Channel updates
       INPUT_CHANNEL = 0xf35aec28
@@ -128,6 +140,18 @@ module MTProto
       UPDATES_DIFFERENCE = 0x00f49ca0
       UPDATES_DIFFERENCE_SLICE = 0xa8fb1981
       UPDATES_DIFFERENCE_TOO_LONG = 0x4afe8f6d
+
+      # The top-level `Updates` type — the only constructors dispatched as updates.
+      UPDATE_CONSTRUCTORS = [
+        UPDATES, UPDATES_COMBINED, UPDATES_TOO_LONG,
+        UPDATE_SHORT, UPDATE_SHORT_MESSAGE, UPDATE_SHORT_CHAT_MESSAGE, UPDATE_SHORT_SENT_MESSAGE
+      ].freeze
+
+      # MTProto service/transport messages that carry no API payload.
+      SERVICE_MESSAGES = [
+        PONG, MSGS_ACK, MSGS_STATE_REQ, MSGS_STATE_INFO, MSGS_ALL_INFO,
+        MSG_DETAILED_INFO, MSG_NEW_DETAILED_INFO, FUTURE_SALT, FUTURE_SALTS
+      ].freeze
     end
   end
 end

data/lib/mtproto/tl/message_entity.rb

@@ -0,0 +1,154 @@
+# frozen_string_literal: true
+
+require_relative 'reader'
+
+module MTProto
+  module TL
+    # A single MessageEntity — one formatting span over a message's text. `offset`
+    # and `length` are in **UTF-16 code units** (Telegram's unit, what the wire
+    # carries), NOT bytes or codepoints. `type` is a symbol; any type-specific
+    # payload rides in the matching field:
+    #   :text_url     -> url
+    #   :pre          -> language
+    #   :mention_name -> user_id (+ access_hash, needed to SEND a text-mention)
+    #   :custom_emoji -> document_id
+    #   :blockquote   -> collapsed (expandable quote)
+    MessageEntity = Struct.new(
+      # :length intentionally shadows Struct#length — it is the span length, the field we want.
+      :type, :offset, :length, :url, :language, :user_id, :access_hash, :document_id, :collapsed, # rubocop:disable Lint/StructNewOverride
+      keyword_init: true
+    ) do
+      def to_h
+        super.compact
+      end
+    end
+
+    # Codec for `Vector<MessageEntity>` — the formatting-span list carried by
+    # messages.sendMessage/editMessage (entities, flags.3) and parsed back out of an
+    # inbound message. `serialize` builds the wire byte array from [MessageEntity];
+    # `parse` reads it back. Layouts follow the layer-227 schema. The only input vs
+    # output split is the text-mention: sending uses inputMessageEntityMentionName
+    # (an InputUser), a received one is messageEntityMentionName (a bare user_id).
+    module MessageEntities
+      extend Binary
+
+      module_function
+
+      VECTOR = 0x1cb5c415
+      INPUT_USER = 0xf21158c6
+      INPUT_USER_SELF = 0xf7c1b13f
+      INPUT_MESSAGE_ENTITY_MENTION_NAME = 0x208e68c9
+
+      # type -> constructor id for the entities that are identical on input/output.
+      TYPE_TO_CTOR = {
+        unknown: 0xbb92ba95, mention: 0xfa04579d, hashtag: 0x6f635b0d,
+        bot_command: 0x6cef8ac7, url: 0x6ed02538, email: 0x64e475c2,
+        bold: 0xbd610bc9, italic: 0x826f8b60, code: 0x28a20571, pre: 0x73924be0,
+        text_url: 0x76a6d327, mention_name: 0xdc7b1140, phone: 0x9b69e34b,
+        cashtag: 0x4c4e743f, underline: 0x9c4e7e8b, strike: 0xbf0693d4,
+        bank_card: 0x761e6af4, spoiler: 0x32ca960f, custom_emoji: 0xc8cf05f8,
+        blockquote: 0xf1ccaaac
+      }.freeze
+      CTOR_TO_TYPE = TYPE_TO_CTOR.invert.freeze
+
+      # [MessageEntity] -> wire byte array for `Vector<MessageEntity>`.
+      def serialize(entities)
+        out = u32_b(VECTOR) + u32_b(entities.length)
+        entities.each { |e| out += serialize_entity(e) }
+        out
+      end
+
+      # Wire byte array for one entity, dispatched on its type. Sending a text-mention
+      # needs an InputUser: an explicit access_hash, or 0 for a user the DC already
+      # knows (e.g. a mutual contact); a wrong hash is rejected by the server.
+      def serialize_entity(entity)
+        return serialize_mention_name(entity) if entity.type == :mention_name
+
+        ctor = TYPE_TO_CTOR.fetch(entity.type) do
+          raise ArgumentError, "unknown entity type: #{entity.type.inspect}"
+        end
+        bytes = u32_b(ctor)
+        bytes += u32_b(entity.collapsed ? 1 : 0) if entity.type == :blockquote # flags.0 collapsed
+        bytes += u32_b(entity.offset) + u32_b(entity.length)
+        case entity.type
+        when :text_url then bytes += tl_string(entity.url.to_s)
+        when :pre then bytes += tl_string(entity.language.to_s)
+        when :custom_emoji then bytes += u64_b(entity.document_id)
+        end
+        bytes
+      end
+
+      # Wire byte array + new offset for one `Vector<MessageEntity>` at `offset`;
+      # returns [[MessageEntity], new_offset]. An entity constructor we don't model is
+      # stepped over via the schema sizer so the walk stays aligned (and dropped).
+      def parse(data, offset)
+        return [[], offset] unless data[offset, 4].unpack1('L<') == VECTOR
+
+        io = offset + 4
+        count = data[io, 4].unpack1('L<')
+        io += 4
+        entities = []
+        count.times do
+          entity, io = parse_entity(data, io)
+          entities << entity if entity
+        end
+        [entities, io]
+      end
+
+      def parse_entity(data, offset)
+        ctor = data[offset, 4].unpack1('L<')
+        type = CTOR_TO_TYPE[ctor]
+        return [nil, Reader.schema.skip(data, offset)] if type.nil?
+
+        io = offset + 4
+        collapsed = nil
+        if type == :blockquote
+          collapsed = data[io, 4].unpack1('L<').anybits?(1 << 0)
+          io += 4
+        end
+        ent_offset = data[io, 4].unpack1('l<')
+        ent_length = data[io + 4, 4].unpack1('l<')
+        io += 8
+        extra = {}
+        case type
+        when :text_url
+          extra[:url], io = Reader.read_tl_string(data, io)
+        when :pre
+          extra[:language], io = Reader.read_tl_string(data, io)
+        when :mention_name
+          extra[:user_id] = data[io, 8].unpack1('q<')
+          io += 8
+        when :custom_emoji
+          extra[:document_id] = data[io, 8].unpack1('q<')
+          io += 8
+        end
+        [MessageEntity.new(type: type, offset: ent_offset, length: ent_length, collapsed: collapsed, **extra), io]
+      end
+
+      # inputMessageEntityMentionName#208e68c9 offset:int length:int user_id:InputUser.
+      def serialize_mention_name(entity)
+        input_user =
+          if entity.user_id.nil?
+            u32_b(INPUT_USER_SELF)
+          else
+            u32_b(INPUT_USER) + u64_b(entity.user_id) + u64_b(entity.access_hash || 0)
+          end
+        u32_b(INPUT_MESSAGE_ENTITY_MENTION_NAME) + u32_b(entity.offset) + u32_b(entity.length) + input_user
+      end
+
+      # TL `string` serializer (byte array): a length-prefixed, 4-byte-padded blob.
+      # Mirrors SendMessage#serialize_tl_string; kept local so the codec stands alone.
+      def tl_string(str)
+        bytes = str.to_s.dup.force_encoding('BINARY').bytes
+        length = bytes.length
+        if length <= 253
+          padding = (4 - ((length + 1) % 4)) % 4
+          [length] + bytes + ([0] * padding)
+        else
+          padding = (4 - ((length + 4) % 4)) % 4
+          [254] + u32_b(length)[0, 3] + bytes + ([0] * padding)
+        end
+      end
+    end
+  end
+end