tg_mq 0.0.3

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: f65f989683ba5ccbb0f3862df8bf42c4e28aeedc5046b2a19fafd0a3a2ffe99f
+  data.tar.gz: b19a7af3a43caf7146825cb8dab01b6148ee8bc570e78ef4d8c7e5418d700e1a
+SHA512:
+  metadata.gz: afd1732c0334d14dd4c827dd9de4efe471b0e36d096c443d527119dd68ee7b270fa7fab33156f5738d7bf0cb8e6e178b23616dc01c0d92ff4586c00347bd2565
+  data.tar.gz: 197bf2b88aaf2550c8e20ca6f31414561a7cf7ef6b1d7037b522e3e1a936dcaf1791b4ab0bca9ea34501956534430c6f430c31095c1fffc14e0ba602e8736676

data/Gemfile

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in aggredator-api.gemspec
+gemspec

data/Gemfile.lock

@@ -0,0 +1,82 @@
+PATH
+  remote: .
+  specs:
+    tg_mq (0.0.1)
+      activesupport (~> 6.0)
+      logger
+      telegram-bot-ruby
+      timeouter
+      zeitwerk
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    activesupport (6.1.7.10)
+      concurrent-ruby (~> 1.0, >= 1.0.2)
+      i18n (>= 1.6, < 2)
+      minitest (>= 5.1)
+      tzinfo (~> 2.0)
+      zeitwerk (~> 2.3)
+    bigdecimal (3.2.2)
+    byebug (12.0.0)
+    concurrent-ruby (1.3.5)
+    dry-core (1.1.0)
+      concurrent-ruby (~> 1.0)
+      logger
+      zeitwerk (~> 2.6)
+    dry-inflector (1.2.0)
+    dry-logic (1.6.0)
+      bigdecimal
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.1)
+      zeitwerk (~> 2.6)
+    dry-struct (1.8.0)
+      dry-core (~> 1.1)
+      dry-types (~> 1.8, >= 1.8.2)
+      ice_nine (~> 0.11)
+      zeitwerk (~> 2.6)
+    dry-types (1.8.2)
+      bigdecimal (~> 3.0)
+      concurrent-ruby (~> 1.0)
+      dry-core (~> 1.0)
+      dry-inflector (~> 1.0)
+      dry-logic (~> 1.4)
+      zeitwerk (~> 2.6)
+    faraday (2.13.1)
+      faraday-net_http (>= 2.0, < 3.5)
+      json
+      logger
+    faraday-multipart (1.1.0)
+      multipart-post (~> 2.0)
+    faraday-net_http (3.4.0)
+      net-http (>= 0.5.0)
+    i18n (1.14.7)
+      concurrent-ruby (~> 1.0)
+    ice_nine (0.11.2)
+    json (2.12.2)
+    logger (1.7.0)
+    minitest (5.25.5)
+    multipart-post (2.4.1)
+    net-http (0.6.0)
+      uri
+    telegram-bot-ruby (2.4.0)
+      dry-struct (~> 1.6)
+      faraday (~> 2.0)
+      faraday-multipart (~> 1.0)
+      zeitwerk (~> 2.6)
+    timeouter (0.1.3.38794)
+    tzinfo (2.0.6)
+      concurrent-ruby (~> 1.0)
+    uri (1.0.3)
+    zeitwerk (2.7.3)
+
+PLATFORMS
+  ruby
+  x86_64-linux
+
+DEPENDENCIES
+  byebug
+  tg_mq!
+
+BUNDLED WITH
+   2.6.2

data/bin/tgmq

@@ -0,0 +1,123 @@
+#!/bin/env ruby
+
+require 'optparse'
+require 'tg_mq'
+
+UTIL = File.basename(__FILE__)
+SELF = File.absolute_path(__FILE__)
+
+Thread.abort_on_exception = true
+
+[STDIN, STDOUT, STDERR].each do |io|
+  io.sync = true
+rescue StandardError
+end
+
+@opts = {
+  itoken: nil,
+  otoken: nil,
+  channel: nil,
+  log_level: ENV.fetch('LOG_LEVEL', ::Logger::INFO)
+}
+
+parser = OptionParser.new do |o|
+  o.banner = "Usage: #{UTIL} [options]"
+
+  o.separator ''
+  o.separator 'TgMq options:'
+
+  o.on('-d', '--debug', 'Enable debug output. Overrides LOG_LEVEL environment variable') do
+    @opts[:log_level] = ::Logger::DEBUG
+  end
+
+  o.on('-s', '--silent', 'Disable info output. Overrides LOG_LEVEL environment variable') do
+    @opts[:log_level] = ::Logger::WARN
+  end
+
+  o.on('-i TOKEN', '--itoken TOKEN', String, 'Telegram Bot token for input channel') do |token|
+    @opts[:itoken] = token.strip
+  end
+
+  o.on('-o TOKEN', '--otoken TOKEN', String, 'Telegram Bot token for output channel') do |token|
+    @opts[:otoken] = token.strip
+  end
+
+  o.on('-c CHANNEL', '--channel CHANNEL', String, 'Telegram Bot chat_id to communicate through') do |channel|
+    @opts[:channel] = channel
+  end
+end
+@args = parser.parse!
+
+TgMq.configure do |config|
+  config.logger.level = @opts[:log_level]
+end
+
+$conn = conn = TgMq::Connection.new(itoken: @opts[:itoken], otoken: @opts[:otoken], channel: @opts[:channel])
+
+control_queue = $queue = queue = SizedQueue.new(50)
+
+def terminate_now
+  $terminate = true
+  STDOUT.flush
+  STDIN.close
+  $conn.stop
+  $queue.push(:terminate)
+end
+
+Signal.trap 'TERM' do
+  $terminate = true
+  Thread.new do
+    warn('Terminating gracefully...')
+    terminate_now
+  end
+end
+
+Signal.trap 'INT' do
+  $terminate = true
+  Thread.new do
+    warn('Stopping gracefully...')
+    terminate_now
+  end
+end
+
+reader = Thread.new(control_queue) do |q, _c|
+  STDIN.binmode
+  loop do
+    q.push([:in, STDIN.readpartial(3000)])
+  rescue EOFError
+    q.push([:in, 'EOF', proc { q.push(:terminate) }])
+    break
+  end
+rescue StandardError
+  q.push(:terminate)
+end
+
+remote = Thread.new(queue, conn) do |q, c|
+  STDOUT.binmode
+  c.subscribe do |line|
+    q.push([:out, line])
+  end
+end
+
+loop do
+  cmd, data, cb = queue.deq(timeout: 1)
+  if cmd == :terminate || $terminate
+    terminate_now
+    break
+  end
+
+  next if cmd.nil?
+
+  if cmd == :in
+    cb ? conn.enqueue_message(data, &cb) : conn.enqueue_message(data)
+  elsif cmd == :out
+    if data.strip == 'EOF'
+      terminate_now
+    else
+      STDOUT.write(data)
+      STDOUT.flush
+    end
+  end
+end
+
+[reader, remote].each(&:join)

data/lib/tg-mq.rb

@@ -0,0 +1 @@
+require_relative './tg_mq'

data/lib/tg_mq/configuration.rb

@@ -0,0 +1,41 @@
+require 'uri'
+
+module TgMq
+  class Configuration
+    attr_accessor :logger
+
+    def initialize(logger = ::Logger.new(STDERR, formatter: Logger::Formatter.new))
+      @logger = logger.respond_to?(:tagged) ? logger : ActiveSupport::TaggedLogging.new(logger)
+    end
+
+    module Concern
+      extend ActiveSupport::Concern
+
+      included do |_klass|
+      end
+
+      class_methods do
+        # Instantiate the Configuration singleton
+        # or return it. Remember that the instance
+        # has attribute readers so that we can access
+        # the configured values
+        def configuration
+          @configuration ||= TgMq::Configuration.new
+        end
+
+        def config
+          configuration
+        end
+
+        # This is the configure block definition.
+        # The configuration method will return the
+        # Configuration singleton, which is then yielded
+        # to the configure block. Then it's just a matter
+        # of using the attribute accessors we previously defined
+        def configure
+          yield(configuration)
+        end
+      end
+    end
+  end
+end

data/lib/tg_mq/connection.rb

@@ -0,0 +1,122 @@
+require 'base64'
+require 'telegram/bot'
+require 'timeouter'
+require 'monitor'
+
+module TgMq
+  class Connection
+    attr_reader :logger
+
+    def initialize(itoken:, otoken:, channel:, logger: TgMq.config.logger)
+      @logger = TgMq.setup_logger(logger).tagged(self.class)
+
+      @mx = Monitor.new
+
+      @itoken = itoken
+      @otoken = otoken
+      @channel = channel
+
+      @output_bot = ::Telegram::Bot::Client.new(@otoken, timeout: 5)
+      @output_shaper = OutputShaper.new(chunk_size: 4080)
+
+      @input_bot = ::Telegram::Bot::Client.new(@itoken, timeout: 5)
+      @input_shaper = InputShaper.new(logger: logger)
+
+      # Telegram Bot api alows 30 rps, we use only 20
+      @delay = 60.0 / 25.0
+      $r = 0
+
+      # @squeue = SizedQueue.new(10)
+      @threads = []
+      @threads << start_sender(@output_shaper, @output_bot, @delay)
+    end
+
+    def enqueue_message(data, &cb)
+      logger.debug("Enqueue message of #{data.size} bytes")
+      @output_shaper.enqueue_data(data, &cb)
+    end
+
+    def stopped?
+      @stopped
+    end
+
+    def stop(timeout = 2)
+      return if @stopped
+
+      @stopped = true
+      @output_bot.stop
+      @input_bot.stop
+
+      if wait_for_termination(timeout)
+        true
+      else
+        @threads.each(&:kill)
+        false
+      end
+    end
+
+    def wait_for_termination(timeout = 0)
+      Timeouter.run(timeout) do |t|
+        return true if @threads.all? do |thread|
+          thread.join(t.left)
+        end
+      end
+    end
+
+    def subscribe(&block)
+      @input_bot.listen do |message|
+        case message
+        when Telegram::Bot::Types::Message
+          logger.debug(123_123)
+          text = (message&.new_chat_title || message&.text || message&.pinned_message&.text).to_s
+          logger.debug "Receive raw: [#{text.first(15)}...] of #{text.size} bytes"
+
+          if (data = @input_shaper.pushdata(text))
+            decoded = Base64.strict_decode64(data)
+            logger.debug "Receive payload: of #{decoded.size} bytes"
+            block.call(decoded)
+          end
+        end
+      end
+    end
+
+    def start_sender(output_shaper, bot, delay)
+      next_send_at = Time.at(0)
+
+      Thread.new(output_shaper, bot) do |c, b|
+        until @stopped
+          break if @stopped
+          next(sleep 1) if Time.now < next_send_at
+
+          frame = c.deq(timeout: 1)
+          break if @stopped
+          next(sleep 1) if frame.nil?
+
+          logger.debug "Sending frame [#{frame.id}] of #{frame.data.size} bytes..."
+
+          response = tgretry { b.api.sendMessage(chat_id: @channel, text: frame.data) }
+          sleep 0.1
+          pin_response = tgretry { b.api.pinChatMessage(chat_id: @channel, message_id: response.message_id) }
+
+          begin
+            frame.callback&.call(pin_response)
+          rescue StandardError
+            nil
+          end
+
+          next_send_at = Time.now + delay * 2
+        end
+      end
+    end
+
+    def tgretry
+      yield
+    rescue Telegram::Bot::Exceptions::ResponseError => e
+      if e.error_code.to_i == 429
+        logger.warn 'Retry packet send in 5 seconds...'
+        sleep 5
+        retry
+      end
+    end
+  end
+end

data/lib/tg_mq/frame.rb

@@ -0,0 +1,40 @@
+module TgMq
+  class Frame
+    BEGIN_MSG = 'tgw'
+    BEGIN_RX = /^#{BEGIN_MSG}\[(?<seq>\d+):(?<msg_id>\d+):(?<num>\d+)\](?<msg>.*)/
+
+    END_MSG = ':wgt'
+    END_RX = /#{END_MSG}$/
+
+    attr_reader :seq, :msg_id, :num, :chunk, :callback
+
+    def initialize(seq, msg_id, num, chunk)
+      @seq = seq
+      @msg_id = msg_id
+      @num = num
+      @chunk = chunk
+    end
+
+    def self.parse(data)
+      if (md = BEGIN_RX.match(data))
+        new(md[:seq], md[:msg_id], md[:num].to_i, md[:msg])
+      end
+    end
+
+    def id
+      "#{message_id}:#{num}"
+    end
+
+    def message_id
+      "#{seq}:#{msg_id}"
+    end
+
+    def data
+      "#{BEGIN_MSG}[#{id}]#{chunk}"
+    end
+
+    def set_callback(&callback)
+      @callback = callback
+    end
+  end
+end

data/lib/tg_mq/input_shaper.rb

@@ -0,0 +1,35 @@
+module TgMq
+  class InputShaper
+    attr_reader :logger
+
+    def initialize(logger: TgMq.config.logger)
+      @logger = TgMq.setup_logger(logger).tagged(self.class)
+
+      @frames = Hash.new do |h, message_id|
+        h[message_id] = {
+          buffer: '',
+          frames: []
+        }
+      end
+    end
+
+    def pushdata(text)
+      if (frame = Frame.parse(text))
+        logger.info "receive frame [#{frame.id}]"
+        frameset = @frames[frame.message_id]
+        data = frameset[:buffer] += frame.chunk
+        frameset[:frames] << frame
+
+        if data.end_with?(Frame::END_MSG)
+          @frames.delete(frame.message_id)
+          return data[0..-5]
+        end
+      else
+        logger.warn "skip [#{text.first(15)}]: not a frame"
+      end
+
+      nil
+    end
+
+  end
+end

data/lib/tg_mq/message.rb

@@ -0,0 +1,28 @@
+module TgMq
+  class Message
+    include Enumerable
+
+    def self.pack(seq, data, msg_id: Time.now.strftime('%L'), chunked: 4080)
+      (data + Frame::END_MSG).scan(/.{1,#{chunked}}/).each_with_index.map do |chunk, idx|
+        Frame.new(seq, msg_id, idx, chunk)
+      end
+    end
+
+    def initialize(seq, chunks, msg_id: Time.now.strftime('%L'))
+      @seq = seq
+      @msg_id = msg_id
+      @chunks = chunks
+    end
+
+    def each
+      if block_given?
+        idx = -1
+        @chunks.each do |chunk|
+          yield(Frame.new(@seq, @msg_id, idx += 1, chunk))
+        end
+      else
+        to_enum(:each)
+      end
+    end
+  end
+end

data/lib/tg_mq/output_shaper.rb

@@ -0,0 +1,37 @@
+module TgMq
+  class OutputShaper
+    attr_reader :chunk_size
+
+    def initialize(chunk_size: 110)
+      @chunk_size = chunk_size
+      @seq = 0
+      @queue = SizedQueue.new(10)
+    end
+
+    def enqueue_data(data, &callback)
+      frames = split_to_frames(next_seq!, data).to_a
+      frames.last&.set_callback(&callback)
+      frames.each do |frame|
+        @queue << frame
+      end
+    end
+
+    def deq(*args, **kwargs)
+      @queue.deq(*args, **kwargs)
+    end
+
+    protected
+
+    def next_seq!
+      @seq += 1
+    end
+
+    def split_to_frames(seq, data, msg_id: Time.now.strftime('%L'))
+      encoded = Base64.strict_encode64(data).strip
+
+      (encoded + Frame::END_MSG).scan(/.{1,#{chunk_size}}/).each_with_index.map do |chunk, idx|
+        Frame.new(seq, msg_id, idx, chunk)
+      end
+    end
+  end
+end

data/lib/tg_mq/version.rb

@@ -0,0 +1,3 @@
+module TgMq
+  VERSION = '0.0.3'
+end

data/lib/tg_mq.rb

@@ -0,0 +1,20 @@
+require 'logger'
+require 'active_support/all'
+
+require 'zeitwerk'
+
+loader = Zeitwerk::Loader.for_gem
+loader.ignore("#{__dir__}/tg-mq.rb")
+loader.inflector.inflect(
+  'tg_mq' => 'TgMq',
+  'tg-mq' => 'TgMq'
+)
+loader.setup
+
+module TgMq
+  include Configuration::Concern
+
+  def self.setup_logger(logger)
+    logger.respond_to?(:tagged) ? logger : ActiveSupport::TaggedLogging.new(logger)
+  end
+end

metadata

@@ -0,0 +1,139 @@
+--- !ruby/object:Gem::Specification
+name: tg_mq
+version: !ruby/object:Gem::Version
+  version: 0.0.3
+platform: ruby
+authors:
+- Samoilenko Yuri
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2025-06-06 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '6.0'
+- !ruby/object:Gem::Dependency
+  name: logger
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: timeouter
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: telegram-bot-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: zeitwerk
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: byebug
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Message Queue backed by Telegram Chat
+email:
+- kinnalru@gmail.com
+executables:
+- tgmq
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- Gemfile.lock
+- bin/tgmq
+- lib/tg-mq.rb
+- lib/tg_mq.rb
+- lib/tg_mq/configuration.rb
+- lib/tg_mq/connection.rb
+- lib/tg_mq/frame.rb
+- lib/tg_mq/input_shaper.rb
+- lib/tg_mq/message.rb
+- lib/tg_mq/output_shaper.rb
+- lib/tg_mq/version.rb
+homepage:
+licenses: []
+metadata: {}
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.4.10
+signing_key:
+specification_version: 4
+summary: Message Queue backed by Telegram Chat
+test_files: []