watchmonkey_cli 1.10.0 → 1.11.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2130727c6200a2fffef5220a52052df322b126e610dfdf18431f4c05841bd286
-  data.tar.gz: e8130e7bd3267c92a3352d102a577d7bd047c35905eae7291e9ddfe49da822e9
+  metadata.gz: 47f837c69b183c6b2f6f1670358d1bd4eb6745afd4db2335e2f1c95749c17022
+  data.tar.gz: bdc02ab27bbb69a4c15eb67c4bc310fbfc25493abb124684353ccddcf7a320e8
 SHA512:
-  metadata.gz: '053523431494e83e9c6e72f4b1d1313cf3be03fe3f5a566deabcc2e2d22ec0db17d7d04f82a0fddbc91cd88024267d36d7a8bd1ffe7b5bc6c1990b6cfd1d41ba'
-  data.tar.gz: d4072ceafae6848acef9e34591cf1e33ddbf6df248eafa8f6018f0c1b7feb1c66e2238b388724a15cd5fa82669b9a6cd21b1f56f0518adc4c0ce040c730074b3
+  metadata.gz: def716ff43830383dd7f095b5c502172ce9ac71f008a9cbee00cd18448aa8d893fdf306e717181e569e70a36facac814433b520674fc0957b27430e48857c6cf
+  data.tar.gz: 1fc92b89105fd547344474405c276661f1383486c5cf0443bc8ee989c991b0fe1a46c003bc88bede5ca3d7e3c91cd0e651e97d82e51d6f984291a5015fe7a63d

data/Gemfile

@@ -2,3 +2,6 @@ source 'https://rubygems.org'
 
 # Specify your gem's dependencies in dle.gemspec
 gemspec
+
+# optional dependencies
+gem 'telegram-bot-ruby'

data/README.md

@@ -4,10 +4,11 @@ Watchmonkey is a very simple tool to monitor resources with Ruby without the nee
 
 Before looking any further you might want to know:
 
-  * There is no escalation or notification system but you may add it yourself
-  * I created this for being used with [Platypus](http://sveinbjorn.org/platypus) hence the [Platypus Hook](https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/platypus.rb)
+  * There is no escalation or notification system (except experimental telegram bot) but you may add it yourself
+  * I originally created this for being used with [Platypus](http://sveinbjorn.org/platypus) hence the [Platypus Hook](https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/platypus.rb)
   * This is how the text output looks like: [Screenshot](http://imgur.com/8yLYnKb)
   * This is how the Platypus support looks like: [ProgressBar](http://imgur.com/Vd8ZD7A) [HTML/WebView](http://imgur.com/5FwmWFZ)
+  * This is how Telegram Bot looks for now: [Telegram Screenshot](http://imgur.com/HBONi51)
 
 ---
 
@@ -43,6 +44,8 @@ To get a list of available options invoke Watchmonkey with the `--help` or `-h`
             --generate-config [myconfig] Generates a example config in ~/.watchmonkey
         -l, --log [file]                 Log to file, defaults to ~/.watchmonkey/logs/watchmonkey.log
         -t, --threads [NUM]              Amount of threads to be used for checking (default: 10)
+        -e, --except tag1,tag2           Don't run tasks tagged with given tags
+        -o, --only tag1,tag2             Only run tasks tagged with given tags
         -s, --silent                     Only print errors and infos
         -q, --quiet                      Only print errors
 
@@ -81,6 +84,15 @@ If you want to monitor something that is not covered by the buildin handlers you
 By default Watchmonkey will run all tests once and then exit. This addon will enable Watchmonkey to run in a loop and run tests on a periodic interval.
 Since this seems like a core feature it might get included directly into Watchmonkey but for now take a look at the [application configuration file](https://github.com/2called-chaos/watchmonkey_cli/blob/master/doc/config_example.rb) and [ReQueue source code](https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/requeue.rb) for integration examples.
 
+### Telegram Bot
+Notify via Telegram. Experimental. Refer to [application configuration file](https://github.com/2called-chaos/watchmonkey_cli/blob/master/doc/config_example.rb) and [TelegramBot source code](https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/telegram_bot.rb) for further information.
+
+* works with ReQueue (wouldn't make much sense otherwise huh?)
+* optional per-user message throttling via checker uniqid (checker name + host + arguments)
+* optional per-user only/except filters based on tags
+* planned: robust telegram connection failure handling
+* planned: per-user regex exclusion filters
+
 ### Platypus support
 [Platypus](http://sveinbjorn.org/platypus) is a MacOS software to create dead simple GUI wrappers for scripts. There is buildin support for the interface types ProgressBar and WebView. For integration examples take a look at the [application configuration file](https://github.com/2called-chaos/watchmonkey_cli/blob/master/doc/config_example.rb) and [Platypus hook source code](https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/platypus.rb).
 

data/VERSION

@@ -1 +1 @@
-1.10.0
+1.11.0

data/doc/checker_example.rb

@@ -23,6 +23,7 @@ module MyWatchmonkeyCheckers
     # Called by configuration defining a check with all the arguments.
     #   e.g. my_checker "http://google.com", some_option: true
     # Should invoke `app.enqueue` which will by default call `#check!` method with given arguments.
+    # Must have options as last argument!
     def enqueue host, opts = {}
       opts = { some_option: false }.merge(opts)
 
@@ -36,6 +37,7 @@ module MyWatchmonkeyCheckers
     end
 
     # First argument is the result object, all other arguments came from `app.enqueue` call.
+    # Must have options as last argument!
     def check! result, host, opts = {}
       # Do your checks and modify the result object.
       # Debug messages will not show if -s/--silent or -q/--quiet argument is passed.

data/doc/config_example.rb

@@ -43,3 +43,59 @@ if @argv.delete("--platypus")
 
   @opts[:colorize] = false # doesn't render in platypus
 end
+
+
+
+# Integrate Telegram notifications
+# For options refer to the source code:
+#     https://github.com/2called-chaos/watchmonkey_cli/blob/master/lib/watchmonkey_cli/hooks/telegram_bot.rb
+if @argv.delete("--telegram")
+  require "watchmonkey_cli/hooks/telegram_bot"
+  WatchmonkeyCli::TelegramBot.hook!(self, {
+    # to create a bot refer to https://core.telegram.org/bots#6-botfather
+    api_key: "123456789:xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx",
+
+    # poll timeout, the longer this is (in seconds) the longer it will take to gracefully shut down
+    timeout: 5,
+
+    # optionally log incoming messages
+    #logger: Logger.new(STDOUT),
+
+    # purge old throttle data, default: 30.days
+    #throttle_retention: 30.days,
+
+    # retry sending messages that failed, default: false
+    # Not recommended since on connection failure a HUGE amount of messages will accumulate
+    # and spam you (and reach rate limits) upon connection restore.
+    #retry_on_egress_failure: false,
+
+    # configure your notification targets, if not listed you can't interact with the bot
+    notify: [
+      [
+        # your telegram ID, if you try talking to the bot it will tell you your ID
+        987654321,
+
+        # flags
+        #   - :all        -- same as :debug, :info, :error (not recommended)
+        #   - :debug      -- send all debug messages (not recommended)
+        #   - :info       -- send all info messages (not recommended)
+        #   - :error      -- send all error messages (RECOMMENDED)
+        #   - :admin_flag -- allows access to some commands (/wm_shutdown /stats)
+        [:error, :admin_flag],
+
+        # options (all optional, you can comment them out but leave the {})
+        {
+          # throttle: seconds(int) -- throttle messages by checker uniqid for this long (0/false = no throttle, default)
+          throttle: 15*60,
+
+          # only: Array(string, symbol) -- only notify when tagged with given tags
+          only: %w[production critical],
+
+          # except: Array(string, symbol) -- don't notify when tagged with given tags (runs after only-check)
+          except: %w[database],
+        }
+      ],
+      [123456789, [:error], { throttle: 30.minutes }]
+    ],
+  })
+end

data/lib/watchmonkey_cli/application.rb

@@ -21,9 +21,22 @@ module WatchmonkeyCli
           app.haltpoint
         rescue Interrupt
           app.abort("Interrupted", 1)
+        rescue SystemExit
+          # silence
         ensure
+          $wm_runtime_exiting = true
           app.fire(:wm_shutdown)
-          app.debug "#{Thread.list.length} threads remain..."
+          if app.filtered_threads.length > 1
+            app.error "[WARN] #{app.filtered_threads.length} threads remain (should be 1)..."
+            app.filtered_threads.each do |thr|
+              app.debug "[THR] #{Thread.main == thr ? "MAIN" : "THREAD"}\t#{thr.alive? ? "ALIVE" : "DEAD"}\t#{thr.inspect}", 10
+              thr.backtrace.each do |l|
+                app.debug "[THR]\t#{l}", 20
+              end
+            end
+          else
+            app.debug "1 thread remains..."
+          end
         end
       end
     end
@@ -52,6 +65,8 @@ module WatchmonkeyCli
         silent: false,           # -s flag
         quiet: false,            # -q flag
         stdout: STDOUT,          # (internal) STDOUT redirect
+        tag_only: [],            # -o flag
+        tag_except: [],          # -e flag
       }
       init_params
       yield(self)
@@ -65,6 +80,8 @@ module WatchmonkeyCli
         opts.on("--generate-config [myconfig]", "Generates a example config in ~/.watchmonkey") {|s| @opts[:dispatch] = :generate_config; @opts[:config_name] = s }
         opts.on("-l", "--log [file]", "Log to file, defaults to ~/.watchmonkey/logs/watchmonkey.log") {|s| @opts[:logfile] = s || logger_filename }
         opts.on("-t", "--threads [NUM]", Integer, "Amount of threads to be used for checking (default: 10)") {|s| @opts[:threads] = s }
+        opts.on("-e", "--except tag1,tag2", Array, "Don't run tasks tagged with given tags") {|s| @opts[:tag_except] = s.map(&:to_sym) }
+        opts.on("-o", "--only tag1,tag2", Array, "Only run tasks tagged with given tags") {|s| @opts[:tag_only] = s.map(&:to_sym) }
         opts.on("-s", "--silent", "Only print errors and infos") { @opts[:silent] = true }
         opts.on("-q", "--quiet", "Only print errors") { @opts[:quiet] = true }
 
@@ -76,10 +93,15 @@ module WatchmonkeyCli
         opts.on("-z", "Do not check for updates on GitHub (with -v/--version)") { @opts[:check_for_updates] = false }
         opts.on("--dump-core", "for developers") { @opts[:dump] = true }
       end
+      fire(:optparse_init, @optparse)
     end
 
     def parse_params
-      @optparse.parse!(@argv)
+      fire(:optparse_parse_before, @optparse)
+      fire(:optparse_parse_around, @optparse) do
+        @optparse.parse!(@argv)
+      end
+      fire(:optparse_parse_after, @optparse)
     rescue OptionParser::ParseError => e
       abort(e.message)
       dispatch(:help)

data/lib/watchmonkey_cli/application/configuration.rb

@@ -62,6 +62,7 @@ module WatchmonkeyCli
       def initialize app, file
         @app = app
         @file = file
+        @tags = []
         begin
           eval File.read(file, encoding: "utf-8"), binding, file
         rescue
@@ -74,9 +75,28 @@ module WatchmonkeyCli
         @app.fetch_connection(:ssh, name, opts, &b)
       end
 
+      def tag_all! *tags
+        @tags = tags.map(&:to_sym)
+      end
+
       def method_missing meth, *args, &block
         if c = @app.checkers[meth.to_s]
-          c.enqueue(*args)
+          opts = args.extract_options!
+          only = @app.opts[:tag_only]
+          except = @app.opts[:tag_except]
+          tags = (@tags + (opts[:tags] || []).map(&:to_sym)).uniq
+          if only.any?
+            if tags.any?{|t| only.include?(t) }
+              if tags.any?{|t| except.include?(t) }
+                return @app.debug "Skipping #{meth} with #{args} and #{opts} due to tag_except filter..."
+              end
+            else
+              return @app.debug "Skipping #{meth} with #{args} and #{opts} due to tag_only filter..."
+            end
+          elsif tags.any?{|t| except.include?(t) }
+            return @app.debug "Skipping #{meth} with #{args} and #{opts} due to tag_except filter..."
+          end
+          c.enqueue(*args, opts.merge(tags: tags))
         else
           super
         end

data/lib/watchmonkey_cli/application/configuration.tpl

@@ -1,5 +1,26 @@
 # This is a Ruby file!
 
+# =========================================
+# = Step 0: Tag all checkers in this file =
+# =========================================
+
+# All checkers you define after this will get these
+# base tags (you can pass additional ones).
+# Do note that you can call this method multiple
+# times and it will replace your base tags for
+# all checkers to follow, base tags will be cleared
+# upon reaching the end of the file.
+
+tag_all! :production, :critical
+
+# You can use additional tags on any checker by passing
+# a `tags` option consisting of an Array of strings or symbols.
+#   e.g.:
+#     some_checker :some_target, tags: %w[critical foo bar]
+
+
+
+
 # =================================
 # = Step 1: Setup SSH connections =
 # =================================

data/lib/watchmonkey_cli/application/core.rb

@@ -1,6 +1,12 @@
 module WatchmonkeyCli
   class Application
     module Core
+      def filtered_threads
+        Thread.list.reject do |thr|
+          thr.backtrace[0]["gems/concurrent-ruby"] rescue false
+        end
+      end
+
       # ===================
       # = Signal trapping =
       # ===================
@@ -37,10 +43,31 @@ module WatchmonkeyCli
         end
       end
 
-      def fire which, *args
+      def fire which, *args, &block
         return if @disable_event_firing
-        sync { debug "[Event] Firing #{which} (#{@hooks[which].try(:length) || 0} handlers) #{args.map(&:class)}", 99 }
-        @hooks[which] && @hooks[which].each{|h| h.call(*args) }
+        sync { debug "[Event] Firing #{which} (#{@hooks[which].try(:length) || 0} handlers) #{args.map(&:class)}#{" HAS_BLOCK" if block}", 99 }
+        if block && (!@hooks[which] || @hooks[which].empty?)
+          block.call
+        else
+          if @hooks[which] && @hooks[which].any?
+            if block
+              catch :abort do
+                _fire_around(@hooks[which], args, 0, &block)
+              end
+            else
+              @hooks[which].all?{|h| h.call(*args) }
+            end
+          end
+        end
+      end
+
+      def _fire_around hooks, args, index = 0, &block
+        return block.call unless hook = hooks[index]
+        skip = catch(:skip) {
+          hook.call(*args) { _fire_around(hooks, args, index + 1, &block) }
+          nil
+        }
+        _fire_around(hooks, args, index + 1, &block) if skip
       end
 
 
@@ -82,8 +109,10 @@ module WatchmonkeyCli
       def close_connections!
         @connections.each do |type, clist|
           clist.each do |id, con|
-            debug "[SHUTDOWN] closing #{type} connection #{id} #{con}"
-            con.close!
+            if con.established?
+              debug "[SHUTDOWN] closing #{type} connection #{id} #{con}"
+              con.close!
+            end
           end
         end
       end
@@ -100,8 +129,14 @@ module WatchmonkeyCli
           @queue << [checker, a, ->(*a) {
             begin
               result = Checker::Result.new(checker, *a)
+
+              # assign tags
+              taskopts = a.extract_options!
+              result.tags = taskopts[:tags] || []
+              a << taskopts
+
               checker.debug(result.str_running)
-              checker.safe(result.str_safe) {
+              checker.rsafe(result) {
                 timeout = checker.class.maxrt.nil? ? @opts[:maxrt] : checker.class.maxrt
                 timeout = timeout.call(self, checker, a) if timeout.respond_to?(:call)
                 begin

data/lib/watchmonkey_cli/application/dispatch.rb

@@ -3,7 +3,11 @@ module WatchmonkeyCli
     module Dispatch
       def dispatch action = (@opts[:dispatch] || :help)
         if respond_to?("dispatch_#{action}")
-          send("dispatch_#{action}")
+          fire(:dispatch_before, action)
+          fire(:dispatch_around, action) do
+            send("dispatch_#{action}")
+          end
+          fire(:dispatch_after, action)
         else
           abort("unknown action #{action}", 1)
         end

data/lib/watchmonkey_cli/application/output_helper.rb

@@ -13,6 +13,10 @@ module WatchmonkeyCli
         sync { @opts[:stdout].send(:warn, *a) }
       end
 
+      def info msg
+        puts c("[INFO]  #{msg}", :blue)
+      end
+
       def debug msg, lvl = 1
         puts c("[DEBUG] #{msg}", :black) if @opts[:debug] && @opts[:debug] >= lvl
       end

data/lib/watchmonkey_cli/checker.rb

@@ -52,16 +52,24 @@ module WatchmonkeyCli
 
     class Result
       attr_reader :checker, :type, :args
-      attr_accessor :result, :command, :data
+      attr_accessor :result, :command, :data, :tags
 
       def initialize checker, *args
         @checker = checker
         @args = args
         @mutex = Monitor.new
         @type = :info
+        @tags = []
         @spool = { error: [], info: [], debug: []}
       end
 
+      def uniqid additional = []
+        ([
+          self.class.name,
+          @args.map(&:to_s).to_s,
+        ] + additional).join("/")
+      end
+
       def sync &block
         @mutex.synchronize(&block)
       end
@@ -197,10 +205,12 @@ module WatchmonkeyCli
             e.backtrace.each{|l| resultobj.debug "\t\t#{l}" }
             resultobj.dump!
           end
-          sleep 1
-          retry
+          unless $wm_runtime_exiting
+            sleep 1
+            retry
+          end
         end
-        resultobj.error! "#{descriptor}retries exceeded"
+        resultobj.error! "retries exceeded"
         resultobj.dump!
       end
     end

data/lib/watchmonkey_cli/checkers/dev_pry.rb

@@ -5,7 +5,7 @@ module WatchmonkeyCli
       self.maxrt = false
 
       def enqueue host, opts = {}
-      	host = app.fetch_connection(:loopback, :local) if !host || host == :local
+        host = app.fetch_connection(:loopback, :local) if !host || host == :local
         host = app.fetch_connection(:ssh, host) if host.is_a?(Symbol)
         app.enqueue(self, host, opts)
       end

data/lib/watchmonkey_cli/hooks/platypus.rb

@@ -71,7 +71,7 @@ module WatchmonkeyCli
                     <dt>Items in Queue</dt><dd class="qlength">#{@queue.length}</dd>
                     <dt>Items in ReQ</dt><dd class="rqlength">#{@requeue.length}</dd>
                     <dt>Workers</dt><dd class="workers">#{@threads.select{|t| t[:working] }.length}/#{@threads.length} working (#{@threads.select(&:alive?).length} alive)</dd>
-                    <dt>Threads</dt><dd class="tlength">#{Thread.list.length}</dd>
+                    <dt>Threads</dt><dd class="tlength">#{filtered_threads.length}</dd>
                     <dt>Processed entries</dt><dd class="processed">#{@processed}</dd>
                     <dt>Watching since</dt><dd>#{@boot}</dd>
                     <dt>Last draw</dt><dd class="lastdraw">#{Time.current}</dd>
@@ -93,7 +93,7 @@ module WatchmonkeyCli
                 $("dd.qlength").html("#{@queue.length}");
                 $("dd.rqlength").html("#{@requeue.length}");
                 $("dd.workers").html("#{@threads.select{|t| t[:working] }.length}/#{@threads.length} working#{ti}");
-                $("dd.tlength").html("#{Thread.list.length}");
+                $("dd.tlength").html("#{filtered_threads.length}");
                 $("dd.processed").html("#{@processed}");
                 $("dd.lastdraw").html("#{Time.current}");
                 $("pre.lasterrors").html("#{escape_javascript @platypus_status_cache[:errors].map{|t,e| "#{t}: #{e}" }.join("\n")}");

data/lib/watchmonkey_cli/hooks/requeue.rb

@@ -37,11 +37,12 @@ module WatchmonkeyCli
               puts "     Queue: #{@queue.length}"
               puts "   Requeue: #{@requeue.length}"
               puts "   Workers: #{@threads.select{|t| t[:working] }.length}/#{@threads.length} working (#{@threads.select(&:alive?).length} alive)"
-              puts "   Threads: #{Thread.list.length}"
+              puts "   Threads: #{filtered_threads.length}"
               # puts "            #{@threads.select(&:alive?).length} alive"
               # puts "            #{@threads.select{|t| t.status == "run" }.length} running"
               # puts "            #{@threads.select{|t| t.status == "sleep" }.length} sleeping"
               puts " Processed: #{@processed}"
+              puts "  Promises: #{@telegram_bot_egress_promises.length}" if @telegram_bot_egress_promises
               puts "========== //STATUS =========="
             end
           end
@@ -101,6 +102,16 @@ module WatchmonkeyCli
             }]
           end
         end
+
+        def requeue_runall
+          return if $wm_runtime_exiting
+          sync do
+            debug "Running all queued tasks immediately!"
+            @requeue.each_with_index do |(run_at, callback), index|
+              @requeue[index][0] = Time.now
+            end
+          end
+        end
       end
     end
   end

data/lib/watchmonkey_cli/hooks/telegram_bot.rb

@@ -0,0 +1,566 @@
+module WatchmonkeyCli
+  class TelegramBot
+    BotProbeError = Class.new(::RuntimeError)
+    UnknownEgressType = Class.new(::RuntimeError)
+
+    class Event
+      attr_reader :app, :bot
+
+      def initialize app, bot, event
+        @app, @bot, @event = app, bot, event
+      end
+
+      def raw
+        @event
+      end
+
+      def method_missing method, *args, &block
+        @event.__send__(method, *args, &block)
+      end
+
+      def user_data
+        app.telegram_bot_get_udata(raw.from.id)
+      end
+
+      def user_admin?
+        user_data && user_data[1].include?(:admin_flag)
+      end
+
+      def from_descriptive
+        "".tap do |name|
+          name << "[BOT] " if from.is_bot
+          name << "#{from.first_name} " if from.first_name.present?
+          name << "#{from.last_name} " if from.last_name.present?
+          name << "(#{from.username}) " if from.username.present?
+          name << "[##{from.id}]"
+        end.strip
+      end
+
+      def message?
+        raw.is_a?(Telegram::Bot::Types::Message)
+      end
+
+      def message
+        return false unless message?
+        raw.text.to_s
+      end
+
+      def chunks
+        @_chunks ||= message.split(" ")
+      end
+
+      def command?
+        raw.is_a?(Telegram::Bot::Types::Message) && message.start_with?("/")
+      end
+
+      def command
+        chunks.first if command?
+      end
+
+      def args
+        chunks[1..-1]
+      end
+
+      def reply msg, msgopts = {}
+        return unless msg.present?
+        msgopts = msgopts.merge(text: msg, chat_id: raw.chat.id)
+        msgopts = msgopts.merge(reply_to_message_id: raw.message_id) unless msgopts[:quote] == false
+        app.telegram_bot_send(msgopts.except(:quote))
+      end
+    end
+
+    class BetterQueue
+      def initialize
+        @stor = []
+        @monitor = Monitor.new
+      end
+
+      def sync &block
+        @monitor.synchronize(&block)
+      end
+
+      [:length, :empty?, :push, :pop, :unshift, :shift, :<<].each do |meth|
+        define_method(meth) do |*args, &block|
+          sync { @stor.send(meth, *args, &block) }
+        end
+      end
+    end
+
+    class PromiseHandler < BetterQueue
+      def scrub!
+        @stor.dup.each_with_index do |p, i|
+          @stor.delete(p) if [:fulfilled, :rejected].include?(p.state)
+        end
+      end
+
+      def await_all!
+        while p = shift
+          p.wait
+        end
+      end
+    end
+
+    def self.hook!(app, bot_opts = {})
+      app.opts[:telegram_bot] = bot_opts.reverse_merge({
+        debug: false,
+        info: true,
+        error: true,
+        throttle_retention: 30.days,
+        retry_on_egress_failure: false,
+        notify: []
+      })
+
+      app.instance_eval do
+        hook :dispatch_around do |action, &act|
+          throw :skip, true unless action == :index
+          begin
+            require "telegram/bot"
+          rescue LoadError
+            abort "[TelegramBot] cannot load telegram/bot, run\n\t# gem install telegram-bot-ruby"
+          end
+          telegram_bot_dispatch(&act)
+        end
+
+        [:debug, :info, :error].each do |level|
+          hook :"on_#{level}" do |msg, robj|
+            telegram_bot_notify(msg, robj)
+          end if app.opts[:telegram_bot][level]
+        end
+
+        hook :wm_shutdown do
+          debug "[TelegramBot] shutting down telegram bot ingress thread"
+          if @telegram_bot_ingress&.alive?
+            @telegram_bot_ingress[:stop] = true
+            @telegram_bot_ingress.join
+          end
+
+          debug "[TelegramBot] shutting down telegram bot egress thread"
+          if @telegram_bot_egress&.alive?
+            telegram_bot_msg_admins "<b>ALERT: Watchmonkey is shutting down!</b>", parse_mode: "HTML" rescue false
+            @telegram_bot_egress_promises.await_all!
+            @telegram_bot_egress[:stop] = true
+            @telegram_bot_egress.join
+          end
+        end
+
+        # --------------------
+
+        def telegram_bot_state
+          "#{wm_cfg_path}/telegram_bot.wmstate"
+        end
+
+        def telegram_bot_dispatch &act
+          parent_thread = Thread.current
+          @telegram_bot_throttle_locks = {}
+          @telegram_bot_egress_promises = PromiseHandler.new
+          @telegram_bot_egress_queue = BetterQueue.new
+          telegram_bot_throttle # eager load
+          telegram_bot_normalize_notify_udata
+
+          # ==================
+          # = ingress thread =
+          # ==================
+          debug "[TelegramBot] Starting telegram bot ingress thread..."
+          @telegram_bot_ingress = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              until Thread.current[:stop]
+                begin
+                  telegram_bot.fetch_updates {|ev| telegram_bot_handle_event(ev) }
+                rescue SocketError, Faraday::ConnectionFailed => ex
+                  error "[TelegramBot] Poll failure, retrying in 3 seconds (#{ex.class}: #{ex.message})"
+                  sleep 3 unless Thread.current[:stop]
+                  retry unless Thread.current[:stop]
+                end
+              end
+            rescue StandardError => ex
+              parent_thread.raise(ex)
+            end
+          end
+
+          # wait for telegram bot to be ready
+          begin
+            Timeout::timeout(10) do
+              loop do
+                break if @telegram_bot_info
+                sleep 0.5
+              end
+            end
+          rescue Timeout::Error
+            abort "[TelegramBot] Failed to start telegram bot within 10 seconds, aborting...", 2
+          end
+
+          # ==================
+          # = egress thread =
+          # ==================
+          debug "[TelegramBot] Starting telegram bot egress thread..."
+          @telegram_bot_egress = Thread.new do
+            Thread.current.abort_on_exception = true
+            begin
+              until Thread.current[:stop] && @telegram_bot_egress_queue.empty?
+                begin
+                  promise, item = @telegram_bot_egress_queue.shift
+                  if item
+                    mode = item.delete(:__mode)
+                    case mode
+                      when :send then promise.set telegram_bot.api.send_message(item)
+                      when :edit then promise.set telegram_bot.api.edit_message_text(item)
+                      else raise(UnknownEgressType, "Unknown egress mode `#{mode}'!")
+                    end
+                  else
+                    @telegram_bot_egress_promises.scrub!
+                    sleep 0.5
+                  end
+                rescue SocketError, Faraday::ConnectionFailed => ex
+                  if opts[:telegram_bot][:retry_on_egress_failure]
+                    error "[TelegramBot] Push failure, retrying in 3 seconds (#{ex.class}: #{ex.message})"
+                    @telegram_bot_egress_queue.unshift([promise, item.merge(__mode: mode)])
+                    sleep 3 unless Thread.current[:stop]
+                  else
+                    error "[TelegramBot] Push failure, discarding message (#{ex.class}: #{ex.message})"
+                    sleep 0.5
+                  end
+                end
+              end
+            rescue StandardError => ex
+              parent_thread.raise(ex)
+            end
+          end
+
+          telegram_bot_msg_admins "<b>INFO: Watchmonkey is watching#{" ONCE" unless opts[:loop_forever]}!</b>", disable_notification: true, parse_mode: "HTML"
+          act.call
+        rescue BotProbeError => ex
+          abort ex.message
+        ensure
+          @telegram_bot_egress_promises.await_all!
+          File.open(telegram_bot_state, "w+") do |f|
+            f << telegram_bot_throttle.to_json
+          end
+        end
+
+        def telegram_bot
+          @telegram_bot ||= begin
+            Telegram::Bot::Client.new(opts[:telegram_bot][:api_key], opts[:telegram_bot].except(:api_key)).tap do |bot|
+              begin
+                me = bot.api.get_me
+                if me["ok"]
+                  @telegram_bot_info = me["result"]
+                  info "[TelegramBot] Established client [#{@telegram_bot_info["id"]}] #{@telegram_bot_info["first_name"]} (#{@telegram_bot_info["username"]})"
+                else
+                  raise BotProbeError, "Failed to get telegram client information, got NOK response (#{me.inspect})"
+                end
+              rescue HTTParty::RedirectionTooDeep, Telegram::Bot::Exceptions::ResponseError => ex
+                raise BotProbeError, "Failed to get telegram client information, API key correct? (#{ex.class})"
+              rescue SocketError, Faraday::ConnectionFailed => ex
+                raise BotProbeError, "Failed to get telegram client information, connection failure? (#{ex.class}: #{ex.message})"
+              end
+            end
+          end
+        end
+
+        def telegram_bot_send payload
+          Concurrent::Promise.new.tap do |promise|
+            @telegram_bot_egress_queue.push([promise, payload.merge(__mode: :send)])
+          end
+        end
+
+        def telegram_bot_edit payload
+          Concurrent::Promise.new.tap do |promise|
+            @telegram_bot_egress_queue.push([promise, payload.merge(__mode: :edit)])
+          end
+        end
+
+        def telegram_bot_tid_exclusive tid, &block
+          sync do
+            @telegram_bot_throttle_locks[tid] ||= Monitor.new
+          end
+          @telegram_bot_throttle_locks[tid].synchronize(&block)
+        end
+
+        def telegram_bot_throttle
+          @telegram_bot_throttle ||= begin
+            Hash.new {|h,k| h[k] = Hash.new }.tap do |h|
+              if File.exist?(telegram_bot_state)
+                json = File.read(telegram_bot_state)
+                if json.present?
+                  JSON.parse(json).each do |k, v|
+                    case k
+                    when "__mute_until"
+                      h[k] = Time.parse(v)
+                    else
+                      v.each do |k, data|
+                        data[0] = Time.parse(data[0])
+                        data[1] = Time.parse(data[1])
+                      end
+                      v.delete_if {|k, data| data[0] < opts[:telegram_bot][:throttle_retention].ago }
+                      h[k.to_i] = v
+                    end
+                  end
+                end
+              end
+            end
+          end
+        end
+
+        def telegram_bot_normalize_notify_udata
+          opts[:telegram_bot][:notify].each do |a|
+            a[1] = (a[1] ? [*a[1]] : [:error]).flat_map(&:to_sym).uniq
+            a[2] ||= {}
+          end
+        end
+
+        def telegram_bot_get_udata lookup_tid
+          opts[:telegram_bot][:notify].detect do |tid, level, topts|
+            tid == lookup_tid
+          end
+        end
+
+        def telegram_bot_user_muted? tid, notify = true
+          telegram_bot_tid_exclusive(tid) do
+            return false unless mute_until = telegram_bot_throttle[tid]["__mute_until"]
+            if mute_until > Time.current
+              return true
+            else
+              telegram_bot_throttle[tid].delete("__mute_until")
+              telegram_bot_send(text: "You have been unmuted (expired #{mute_until})", chat_id: tid, disable_notification: true) if notify
+              return false
+            end
+          end
+        end
+
+        def telegram_bot_notify msg, robj
+          to_notify = opts[:telegram_bot][:notify].select do |tid, level, topts|
+            level.include?(:all) || level.include?(robj.type)
+          end
+
+          to_notify.each do |tid, level, topts|
+            telegram_bot_tid_exclusive(tid) do
+              next if telegram_bot_user_muted?(tid)
+              if robj
+                # gate only tags
+                next if topts[:only] && topts[:only].any? && !robj.tags.any?{|t| topts[:only].include?(t) }
+
+                # gate except tags
+                next if topts[:except] && topts[:except].any? && robj.tags.any?{|t| topts[:except].include?(t) }
+
+                # send message
+                throttle = telegram_bot_throttle[tid][robj.uniqid] ||= [Time.current, Time.current, 0, nil]
+                throttle[2] += 1
+                if (Time.current - throttle[0]) <= (topts[:throttle] || 0) && throttle[3]
+                  throttle[1] = Time.current
+                  _telegram_bot_sendmsg(tid, msg, throttle[2], throttle, throttle[3])
+                else
+                  throttle[0] = Time.current
+                  throttle[2] = 1
+                  throttle[3] = nil
+                  _telegram_bot_sendmsg(tid, msg, throttle[2], throttle)
+                end
+              else
+                _telegram_bot_sendmsg(tid, msg, 0, [])
+              end
+            end
+          end
+        end
+
+        def _telegram_bot_sendmsg tid, msg, repeat, result_to, msgid = nil
+          msg = "<pre>#{ERB::Util.h msg}</pre>"
+          msg = "#{msg}\n<b>(message repeated #{repeat} times)</b> #{"    <i>last occurance: #{result_to[1]}</i>" if result_to[1]}" if repeat > 1 && msgid
+          (msgid ?
+            telegram_bot_edit(chat_id: tid, message_id: msgid, text: msg, disable_web_page_preview: true, parse_mode: "html")
+            :
+            telegram_bot_send(chat_id: tid, text: msg, disable_web_page_preview: true, parse_mode: "html")
+          ).tap do |promise|
+            await = Concurrent::Promise.new
+            promise.on_success do |m|
+              if msgid = m.dig("result", "message_id")
+                result_to[3] = msgid
+                await.set :success
+              else
+                puts m.inspect
+                await.set :failed
+              end
+            end
+            promise.on_error do |ex|
+              error "[TelegramBot] MessagePromiseError - #{ex.class}:#{ex.message}"
+              await.set :failed
+            end
+            @telegram_bot_egress_promises << await
+          end
+        rescue
+        end
+
+        def _telegram_bot_timestr_parse *chunks
+          time = Time.current
+          chunks.flatten.each do |chunk|
+            if chunk.end_with?("d")
+              time += chunk[0..-2].to_i.days
+            elsif chunk.end_with?("h")
+              time += chunk[0..-2].to_i.hours
+            elsif chunk.end_with?("m")
+              time += chunk[0..-2].to_i.minutes
+            elsif chunk.end_with?("s")
+              time += chunk[0..-2].to_i.seconds
+            else
+              time += chunk.to_i.seconds
+            end
+          end
+          time
+        end
+
+        def telegram_bot_handle_event ev
+          Event.new(self, telegram_bot, ev).tap do |event|
+            begin
+              ctrl = catch :event_control do
+                case event.command
+                when "/ping"
+                  event.reply "Pong!"
+                  throw :event_control, :done
+                when "/start"
+                  if event.user_data
+                    event.reply [].tap{|m|
+                      m << "<b>Welcome!</b> I will tell you if something is wrong with bmonkeys infrastructure."
+                      m << "Your current tags are: #{event.user_data[1].join(", ")}"
+                      m << "<b>You have admin permissions!</b>" if event.user_admin?
+                      m << "\nInstead of muting me in Telegram you can silence me for a while with <code>/mute 6h 30m</code>."
+                      m << "Use <code>/help</code> for more information."
+                    }.join("\n"), parse_mode: "HTML"
+                  else
+                    event.reply %{
+                      Hello there, unfortunately I don't recognize your user id (#{event.from.id}).
+                      Please ask your admin to add you to the configuration file.
+                    }
+                  end
+                  throw :event_control, :done
+                when "/mute"
+                  if event.user_data
+                    telegram_bot_tid_exclusive(event.from.id) do
+                      telegram_bot_user_muted?(event.from.id) # clears if expired
+                      if event.args.any?
+                        mute_until = _telegram_bot_timestr_parse(event.args)
+                        telegram_bot_throttle[event.from.id]["__mute_until"] = mute_until
+                        event.reply "You are muted until #{mute_until}\nUse /unmute to prematurely cancel the mute.", disable_notification: true
+                      else
+                        msg = "Usage: <code>/mute &lt;1d 2h 3m 4s&gt;</code>"
+                        if mute_until = telegram_bot_throttle[event.from.id]["__mute_until"]
+                          msg << "\n<b>You are currently muted until #{mute_until}</b> /unmute"
+                        end
+                        event.reply msg, parse_mode: "HTML", disable_notification: true
+                      end
+                    end
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                when "/unmute"
+                  if event.user_data
+                    telegram_bot_tid_exclusive(event.from.id) do
+                      if mute_until = telegram_bot_throttle[event.from.id]["__mute_until"]
+                        event.reply "You have been unmuted (prematurely canceled #{mute_until})", disable_notification: true
+                      else
+                        event.reply "You are not currently muted.", disable_notification: true
+                      end
+                    end
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                when "/scan", "/rescan"
+                  if event.user_data
+                    if respond_to?(:requeue_runall)
+                      event.reply "Triggering all tasks in ReQueue…", disable_notification: true
+                      requeue_runall
+                    else
+                      event.reply "Watchmonkey is not running with ReQueue!", disable_notification: true
+                    end
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                when "/clearthrottle", "/clearthrottles"
+                  if event.user_data
+                    telegram_bot_tid_exclusive(event.from.id) do
+                      was_muted = telegram_bot_throttle[event.from.id].delete("__mute_until")
+                      telegram_bot_throttle.delete(event.from.id)
+                      event.reply "Cleared all your throttles!", disable_notification: true
+                      if was_muted
+                        telegram_bot_throttle[event.from.id]["__mute_until"] = was_muted
+                      end
+                    end
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                when "/status"
+                  if event.user_admin?
+                    msg = []
+                    msg << "<pre>"
+                    msg << "==========  STATUS  =========="
+                    msg << "     Queue: #{@queue.length}"
+                    msg << "   Requeue: #{@requeue.length}" if @requeue
+                    msg << "   Workers: #{@threads.select{|t| t[:working] }.length}/#{@threads.length} working (#{@threads.select(&:alive?).length} alive)"
+                    msg << "   Threads: #{filtered_threads.length}"
+                    msg << " Processed: #{@processed}"
+                    msg << "  Promises: #{@telegram_bot_egress_promises.length}"
+                    msg << "========== //STATUS =========="
+                    msg << "</pre>"
+                    event.reply msg.join("\n"), parse_mode: "HTML", disable_notification: true
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                when "/help"
+                  event.reply [].tap{|m|
+                    m << "<b>/start</b> Bot API"
+                    if event.user_data
+                      m << "<b>/mute &lt;time&gt;</b> Don't send messages for this long (e.g. <code>/mute 1d 2h 3m 4s</code>)"
+                      m << "<b>/unmute</b> Cancel mute"
+                      m << "<b>/scan</b> Trigger all queued scans in ReQueue"
+                      m << "<b>/clearthrottle</b> Clears your throttled messages"
+                    end
+                    if event.user_admin?
+                      m << "<b>/status</b> Some status information"
+                      m << "<b>/wm_shutdown</b> Shutdown watchmonkey process, may respawn if deamonized"
+                    end
+                  }.join("\n"), parse_mode: "HTML", disable_notification: true
+                  throw :event_control, :done
+                when "/wm_shutdown"
+                  if event.user_admin?
+                    $wm_runtime_exiting = true
+                    telegram_bot_msg_admins "ALERT: `#{event.from_descriptive}` invoked shutdown!"
+                    throw :event_control, :done
+                  else
+                    throw :event_control, :access_denied
+                  end
+                end
+              end
+
+              if ctrl == :access_denied
+                event.reply("You don't have sufficient permissions to execute this command!")
+              elsif ctrl == :bad_request
+                event.reply("Bad Request: Your input is invalid!")
+              end
+            rescue StandardError => ex
+              event.reply("Sorry, encountered an error while processing your request!")
+
+              # notify admins
+              msg = "ERROR: Encountered error while processing a request!\n"
+              msg << "Request: #{ERB::Util.h event.message}\n"
+              msg << "Origin: #{ERB::Util.h event.from.inspect}\n"
+              msg << "<pre>#{ERB::Util.h ex.class}: #{ERB::Util.h ex.message}\n#{ERB::Util.h ex.backtrace.join("\n")}</pre>"
+              telegram_bot_msg_admins(msg, parse_mode: "HTML")
+            end
+          end
+        end
+
+        def telegram_bot_msg_admins msg, msgopts = {}
+          return unless msg.present?
+          opts[:telegram_bot][:notify].each do |tid, level, topts|
+            next unless level.include?(:admin_flag)
+            next if telegram_bot_user_muted?(tid)
+            telegram_bot_send(msgopts.merge(text: msg, chat_id: tid))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/watchmonkey_cli/loopback_connection.rb

@@ -1,7 +1,10 @@
 module WatchmonkeyCli
   class LoopbackConnection
+    attr_reader :opts, :established
+
     def initialize(id, opts = {}, &initializer)
       @id = id
+      @established = false
       @opts = {}.merge(opts)
       # @mutex = Monitor.new
       initializer.try(:call, @opts)
@@ -15,6 +18,10 @@ module WatchmonkeyCli
       "lo:#{@id}"
     end
 
+    def established?
+      @established
+    end
+
     def sync &block
       # @mutex.synchronize(&block)
       block.try(:call)

data/lib/watchmonkey_cli/ssh_connection.rb

@@ -4,6 +4,7 @@ module WatchmonkeyCli
 
     def initialize(id, opts = {}, &initializer)
       @id = id
+      @established = false
 
       if opts.is_a?(String)
         u, h = opts.split("@", 2)
@@ -30,6 +31,10 @@ module WatchmonkeyCli
       "ssh:#{@id}"
     end
 
+    def established?
+      @established
+    end
+
     def sync &block
       @mutex.synchronize(&block)
     end
@@ -42,7 +47,12 @@ module WatchmonkeyCli
     end
 
     def connection
-      sync { @ssh ||= Net::SSH.start(nil, nil, @opts) }
+      sync {
+        @ssh ||= begin
+          @established = true
+          Net::SSH.start(nil, nil, @opts)
+        end
+      }
     end
 
     def close!

data/lib/watchmonkey_cli/version.rb

@@ -1,4 +1,4 @@
 module WatchmonkeyCli
-  VERSION = "1.10.0"
+  VERSION = "1.11.0"
   UPDATE_URL = "https://raw.githubusercontent.com/2called-chaos/watchmonkey_cli/master/VERSION"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: watchmonkey_cli
 version: !ruby/object:Gem::Version
-  version: 1.10.0
+  version: 1.11.0
 platform: ruby
 authors:
 - Sven Pachnit
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2020-01-21 00:00:00.000000000 Z
+date: 2020-04-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -141,6 +141,7 @@ files:
 - lib/watchmonkey_cli/helper.rb
 - lib/watchmonkey_cli/hooks/platypus.rb
 - lib/watchmonkey_cli/hooks/requeue.rb
+- lib/watchmonkey_cli/hooks/telegram_bot.rb
 - lib/watchmonkey_cli/loopback_connection.rb
 - lib/watchmonkey_cli/ssh_connection.rb
 - lib/watchmonkey_cli/version.rb