agents_control 0.3.0 → 0.3.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 7cbdfdaec647d99052be4572b93c02e8a1ba8f1169219a85c3abebb1ce1d794b
-  data.tar.gz: 5d486d2543a88f451eec31e8261ddf8fafb8170446f09f1f13596b07e639f19a
+  metadata.gz: 5b56040135b5403eefcaaa60ef384fcba2f5cb9bf28add127443bfbe741df2ee
+  data.tar.gz: c19ff5fd9a1dfd0cb46373b2ec58e66b095108072ca18efbb3342f9a3c34f121
 SHA512:
-  metadata.gz: e47eca628d62039dd339691b713ad2b123f5fe9205b0394f623fe51806cbb0711b3ee93f5708548855049d6ebf3631776276e8f21b3b977e776392f7bb842be1
-  data.tar.gz: cdec6b32aade79e9b165c72ac9fc121116b1a90ab1c7e16066737561653c9e5090ab87b593e55da04ca78dbe02099a4903bc626b008fe16170a448a65ab8ecf3
+  metadata.gz: 73cbb9c93c0a7b506ed52e51811c9ed2462276c59a3e10f37747054fc05be8b2b2fed83aec1110a151b4ca65470673b7ec65791d5e2d400cd3199cd448306e75
+  data.tar.gz: 6825f34afc030b952b5bc6690170241844888b71e5270fa057597f236a0e2869da1c1caff6cfb4563faee28d84a403d9775fa434d780d672057bc1a1e4dc8ea1

data/lib/agents_control/channels/telegram/api.rb

@@ -242,10 +242,16 @@ module AgentsControl
           raise NetworkError, e.message
         end
 
+        # SystemCallError as a family rather than a list of Errno
+        # constants. The list was a bet on knowing every errno a network
+        # stack can raise, and it lost: Errno::EADDRNOTAVAIL — "can't
+        # assign requested address", what a VPN reconnecting produces —
+        # wasn't on it, so it escaped as itself instead of arriving as
+        # Unavailable, and killed the polling thread outright. The
+        # daemon stayed up with its port open and simply stopped hearing
+        # Telegram, for days, with nothing in the log after the trace.
         NETWORK_ERRORS = [
-          Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::EHOSTUNREACH,
-          Errno::ENETUNREACH, Errno::ETIMEDOUT, Errno::EPIPE,
-          SocketError, Timeout::Error, OpenSSL::SSL::SSLError, IOError
+          SystemCallError, SocketError, Timeout::Error, OpenSSL::SSL::SSLError, IOError
         ].freeze
       end
     end

data/lib/agents_control/channels/telegram/bot.rb

@@ -16,6 +16,19 @@ module AgentsControl
         # closed laptop doesn't hammer reconnects all night.
         BACKOFF = [1, 2, 5, 10, 30, 60].freeze
 
+        # A successful poll leaves a timestamp behind, so something
+        # outside this process can tell a bot that is listening from one
+        # that stopped hours ago. `polling?` answers that in-process, and
+        # doctor runs as a separate command — the daemon's port reports
+        # only that it is up, which a deaf daemon is too.
+        HEARTBEAT_KEY = "telegram:last_poll"
+
+        # A healthy poll returns at least every telegram.poll_timeout
+        # (30s), and immediately when something arrives. Writing on each
+        # one would rewrite the store several times a second under
+        # traffic, for a reading nobody needs that finely.
+        HEARTBEAT_EVERY = 15
+
         def initialize(api:, router:, store:, config:, logger: $stdout)
           @api = api
           @router = router
@@ -59,6 +72,12 @@ module AgentsControl
 
         def stop = @stopped&.push(nil)
 
+        # Whether the Telegram side is actually alive. The daemon holds
+        # its port and answers hooks whether or not this thread is
+        # running, so without asking there is no way to tell a bot that
+        # is listening from one that died quietly hours ago.
+        def polling? = @poller&.alive? || false
+
         def running? = @running
 
         private
@@ -69,6 +88,7 @@ module AgentsControl
           while @running
             begin
               poll_once
+              heartbeat
               failures = 0
             rescue Api::Conflict => e
               # Another process is reading updates — looping further is
@@ -86,10 +106,41 @@ module AgentsControl
               failures += 1
               log("API error: #{e.message}")
               pause(backoff(failures))
+            rescue StandardError => e
+              # A backstop, not a place to add expected errors above it.
+              # Whatever it is, the bot going deaf until someone notices
+              # is worse than retrying: the daemon keeps its port and its
+              # hooks either way, so a dead poller looks exactly like a
+              # healthy one from outside.
+              failures += 1
+              log("unexpected #{e.class}: #{redacted(e)}, retrying in #{backoff(failures)}s")
+              pause(backoff(failures))
             end
           end
         end
 
+        # The message may carry the token, so it is only printed once
+        # redaction has actually succeeded — and redaction failing must
+        # not take the handler down with it, which is how the backstop
+        # above managed to kill the very thread it exists to keep alive.
+        def redacted(error)
+          @api.redact(error.message)
+        rescue StandardError
+          "(message withheld — could not redact)"
+        end
+
+        # Best-effort by design: a store that can't be written is not a
+        # reason to stop reading Telegram, which is the whole job.
+        def heartbeat
+          now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          return if @last_heartbeat && now - @last_heartbeat < HEARTBEAT_EVERY
+
+          @last_heartbeat = now
+          @store.put(Time.now.to_i, ttl: 86_400, key: HEARTBEAT_KEY)
+        rescue StandardError => e
+          log("couldn't record the poll heartbeat: #{e.class}")
+        end
+
         def poll_once
           updates = @api.get_updates(offset: offset, timeout: @config.get("telegram.poll_timeout", 30))
 

data/lib/agents_control/doctor.rb

@@ -18,16 +18,17 @@ module AgentsControl
     # control the app.
     NOT_AUTHORISED = "-1743"
 
-    def initialize(config: nil, secrets: nil, executor: Executor.new, api: nil)
+    def initialize(config: nil, secrets: nil, executor: Executor.new, api: nil, store: nil)
       @config = config || Config.load
       @secrets = secrets || Secrets.new
       @executor = executor
       @api = api
+      @store = store
     end
 
     CHECKS = %i[
       ruby_check terminal_check automation_check agent_binary_check
-      token_check chats_check passphrase_check bot_check daemon_check hooks_check
+      token_check chats_check passphrase_check bot_check daemon_check polling_check hooks_check
       anchors_check wake_check
     ].freeze
 
@@ -185,6 +186,36 @@ module AgentsControl
       end
     end
 
+    # The daemon holds its port and answers hooks whether or not it is
+    # still reading Telegram, so "running" and "listening" are separate
+    # questions and only the first was ever asked. A poll loop that died
+    # on a network error looked exactly like a healthy one — for days,
+    # in the case that prompted this.
+    # Derived from the configured long-poll timeout rather than fixed:
+    # a healthy poll returns at least that often, so a hand-raised
+    # telegram.poll_timeout would otherwise make a perfectly good daemon
+    # read as deaf. Two windows plus a minute leaves room for a retry
+    # and the heartbeat's own throttle.
+    def stale_after
+      [(@config.get("telegram.poll_timeout", 30).to_i * 2) + 60, 120].max
+    end
+
+    def polling_check
+      return nil unless daemon_response(@config.get("hooks.port", Daemon::DEFAULT_PORT)) == :ours
+
+      last = store.get(Channels::Telegram::Bot::HEARTBEAT_KEY)
+      return warn("Telegram", "the daemon hasn't reported a poll yet",
+                  "if it persists, restart: agents_control stop && agents_control") unless last
+
+      age = Time.now.to_i - last.to_i
+      return ok("Telegram", "polling — last heard #{age}s ago") if age < stale_after
+
+      fail("Telegram", "the daemon is up but hasn't polled for #{age / 60} min — messages aren't arriving",
+           "agents_control stop && agents_control")
+    end
+
+    def store = @store ||= Store.new
+
     def daemon_response(port)
       socket = Socket.tcp("127.0.0.1", port, connect_timeout: 1)
       socket.print("POST /ping HTTP/1.1\r\nHost: localhost\r\nContent-Length: 2\r\n\r\n{}")

data/lib/agents_control/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module AgentsControl
-  VERSION = "0.3.0"
+  VERSION = "0.3.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: agents_control
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.3.1
 platform: ruby
 authors:
 - Sapar Kurmanov