kennel 1.74.0

data/lib/kennel/github_reporter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+# Not used in here, but in our templated repo ... so keeping it around for now.
+module Kennel
+  class GithubReporter
+    MAX_COMMENT_SIZE = 65536
+    TRUNCATED_MSG = "\n```\n... (truncated)" # finish the code block so it look nice
+
+    class << self
+      def report(token, &block)
+        return yield unless token
+        new(token, Utils.capture_sh("git rev-parse HEAD").strip).report(&block)
+      end
+    end
+
+    def initialize(token, git_sha)
+      @token = token
+      @git_sha = git_sha
+      origin = ENV["PROJECT_REPOSITORY"] || Utils.capture_sh("git remote -v").split("\n").first
+      @repo_part = origin[%r{github\.com[:/](.+?)(\.git|$)}, 1] || raise("no origin found")
+    end
+
+    def report(&block)
+      output = Utils.strip_shell_control(Utils.tee_output(&block).strip)
+    rescue StandardError
+      output = "Error:\n#{$ERROR_INFO.message}"
+      raise
+    ensure
+      comment "```\n#{output || "Error"}\n```"
+    end
+
+    # https://developer.github.com/v3/repos/comments/#create-a-commit-comment
+    def comment(body)
+      # truncate to maximum allowed comment size for github to avoid 422
+      if body.bytesize > MAX_COMMENT_SIZE
+        body = body.byteslice(0, MAX_COMMENT_SIZE - TRUNCATED_MSG.bytesize) + TRUNCATED_MSG
+      end
+
+      post "commits/#{@git_sha}/comments", body: body
+    end
+
+    private
+
+    def post(path, data)
+      url = "https://api.github.com/repos/#{@repo_part}/#{path}"
+      response = Faraday.post(url, data.to_json, authorization: "token #{@token}")
+      raise "failed to POST to github:\n#{url} -> #{response.status}\n#{response.body}" unless response.status == 201
+    end
+  end
+end

data/lib/kennel/importer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Kennel
+  class Importer
+    TITLES = [:name, :title].freeze
+    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Syncer::TRACKING_FIELDS, :template_variables].freeze
+
+    def initialize(api)
+      @api = api
+    end
+
+    def import(resource, id)
+      if ["screen", "dash"].include?(resource)
+        raise ArgumentError, "resource 'screen' and 'dash' are deprecated, use 'dashboard'"
+      end
+
+      model =
+        begin
+          Kennel::Models.const_get(resource.capitalize)
+        rescue NameError
+          raise ArgumentError, "#{resource} is not supported"
+        end
+
+      data = @api.show(model.api_resource, id)
+      id = data.fetch(:id) # keep native value
+      model.normalize({}, data) # removes id
+      data[:id] = id
+
+      title_field = TITLES.detect { |f| data[f] }
+      title = data.fetch(title_field)
+      title.tr!(Kennel::Models::Record::LOCK, "") # avoid double lock icon
+
+      # calculate or reuse kennel_id
+      # TODO: this is copy-pasted from syncer, need to find a nice way to reuse it
+      tracking_field = Syncer::TRACKING_FIELDS.detect { |f| data[f] }
+      data[:kennel_id] =
+        if tracking_field && data[tracking_field].sub!(/\n?-- Managed by kennel (\S+:\S+).*/, "")
+          $1.split(":").last
+        else
+          Kennel::Utils.parameterize(title)
+        end
+
+      case resource
+      when "monitor"
+        # flatten monitor options so they are all on the base
+        data.merge!(data.delete(:options))
+        data.merge!(data.delete(:thresholds) || {})
+        [:notify_no_data, :notify_audit].each { |k| data.delete(k) if data[k] } # monitor uses true by default
+        data = data.slice(*model.instance_methods)
+
+        # make query use critical method if it matches
+        critical = data[:critical]
+        query = data[:query]
+        if query && critical
+          query.sub!(/([><=]) (#{Regexp.escape(critical.to_f.to_s)}|#{Regexp.escape(critical.to_i.to_s)})$/, "\\1 \#{critical}")
+        end
+
+        data[:type] = "query alert" if data[:type] == "metric alert"
+      when "dashboard"
+        widgets = data[:widgets]&.flat_map { |widget| widget.dig(:definition, :widgets) || [widget] }
+        widgets&.each { |widget| dry_up_query!(widget) }
+      end
+
+      # simplify template_variables to array of string when possible
+      if vars = data[:template_variables]
+        vars.map! { |v| v[:default] == "*" && v[:prefix] == v[:name] ? v[:name] : v }
+      end
+
+      pretty = pretty_print(data).lstrip.gsub("\\#", "#")
+      <<~RUBY
+        #{model.name}.new(
+          self,
+          #{pretty}
+        )
+      RUBY
+    end
+
+    private
+
+    # reduce duplication in imports by using dry `q: :metadata` when possible
+    def dry_up_query!(widget)
+      (widget.dig(:definition, :requests) || []).each do |request|
+        next unless request.is_a?(Hash)
+        next unless metadata = request[:metadata]
+        next unless query = request[:q]&.dup
+        metadata.each do |m|
+          next unless exp = m[:expression]
+          query.sub!(exp, "")
+        end
+        request[:q] = :metadata if query.delete(", ") == ""
+      end
+    end
+
+    def pretty_print(hash)
+      sort_widgets hash
+
+      sort_hash(hash).map do |k, v|
+        pretty_value =
+          if v.is_a?(Hash) || (v.is_a?(Array) && !v.all? { |e| e.is_a?(String) })
+            # update answer here when changing https://stackoverflow.com/questions/8842546/best-way-to-pretty-print-a-hash
+            # (exclude last indent gsub)
+            pretty = JSON.pretty_generate(v)
+              .gsub(": null", ": nil")
+              .gsub(/(^\s*)"([a-zA-Z][a-zA-Z\d_]*)":/, "\\1\\2:") # "foo": 1 -> foo: 1
+              .gsub(/: \[\n\s+\]/, ": []") # empty arrays on a single line
+              .gsub(/^/, "    ") # indent
+              .gsub('q: "metadata"', "q: :metadata") # bring symbols back
+
+            "\n#{pretty}\n  "
+          elsif k == :message
+            "\n    <<~TEXT\n#{v.each_line.map { |l| l.strip.empty? ? "\n" : "      #{l}" }.join}\n      \#{super()}\n    TEXT\n  "
+          elsif k == :tags
+            " super() + #{v.inspect} "
+          else
+            " #{v.inspect} "
+          end
+        "  #{k}: -> {#{pretty_value}}"
+      end.join(",\n")
+    end
+
+    # sort dashboard widgets + nesting
+    def sort_widgets(outer)
+      outer[:widgets]&.each do |widgets|
+        definition = widgets[:definition]
+        definition.replace sort_hash(definition)
+        sort_widgets definition
+      end
+    end
+
+    # important to the front and rest deterministic
+    def sort_hash(hash)
+      Hash[hash.sort_by { |k, _| [SORT_ORDER.index(k) || 999, k] }]
+    end
+  end
+end

data/lib/kennel/models/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require "hashdiff"
+
+module Kennel
+  module Models
+    class Base
+      extend SubclassTracking
+      include SettingsAsMethods
+
+      SETTING_OVERRIDABLE_METHODS = [:name, :kennel_id].freeze
+
+      def kennel_id
+        name = self.class.name
+        if name.start_with?("Kennel::")
+          raise_with_location ArgumentError, "Set :kennel_id"
+        end
+        @kennel_id ||= Utils.snake_case name
+      end
+
+      def name
+        self.class.name
+      end
+
+      def to_json # rubocop:disable Lint/ToJSON
+        raise NotImplementedError, "Use as_json"
+      end
+    end
+  end
+end

data/lib/kennel/models/dashboard.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Dashboard < Record
+      include TemplateVariables
+      include OptionalValidations
+
+      API_LIST_INCOMPLETE = true
+      DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
+      READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
+        :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
+      ]
+      REQUEST_DEFAULTS = {
+        style: { line_width: "normal", palette: "dog_classic", line_type: "solid" }
+      }.freeze
+      WIDGET_DEFAULTS = {
+        "timeseries" => { show_legend: false, legend_size: "0" },
+        "note" => { background_color: "white", font_size: "14", show_tick: false, tick_edge: "left", tick_pos: "50%", text_align: "left" }
+      }.freeze
+      SUPPORTED_DEFINITION_OPTIONS = [:events, :markers, :precision].freeze
+
+      DEFAULTS = {
+        template_variable_presets: nil
+      }.freeze
+
+      settings :title, :description, :definitions, :widgets, :layout_type, :template_variable_presets
+
+      defaults(
+        description: -> { "" },
+        definitions: -> { [] },
+        widgets: -> { [] },
+        template_variable_presets: -> { DEFAULTS.fetch(:template_variable_presets) },
+        id: -> { nil }
+      )
+
+      class << self
+        def api_resource
+          "dashboard"
+        end
+
+        def normalize(expected, actual)
+          super
+
+          ignore_default(expected, actual, DEFAULTS)
+
+          widgets_pairs(expected, actual).each do |pair|
+            # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
+            pair.each do |widgets|
+              widgets.each do |widget|
+                if formats = widget.dig(:definition, :conditional_formats)
+                  widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
+                end
+              end
+            end
+
+            ignore_widget_defaults pair
+
+            ignore_request_defaults(*pair)
+
+            # ids are kinda random so we always discard them
+            pair.each { |widgets| widgets.each { |w| w.delete(:id) } }
+          end
+        end
+
+        private
+
+        def ignore_widget_defaults(pair)
+          pair.map(&:size).max.times do |i|
+            types = pair.map { |w| w.dig(i, :definition, :type) }.uniq
+            next unless types.size == 1
+            next unless defaults = WIDGET_DEFAULTS[types.first]
+            ignore_defaults(pair[0], pair[1], defaults, nesting: :definition)
+          end
+        end
+
+        # discard styles/conditional_formats/aggregator if nothing would change when we applied (both are default or nil)
+        def ignore_request_defaults(expected, actual)
+          [expected.size, actual.size].max.times do |i|
+            a_r = actual.dig(i, :definition, :requests) || []
+            e_r = expected.dig(i, :definition, :requests) || []
+            ignore_defaults e_r, a_r, REQUEST_DEFAULTS
+          end
+        end
+
+        def ignore_defaults(expected, actual, defaults, nesting: nil)
+          [expected.size, actual.size].max.times do |i|
+            e = expected.dig(i, *nesting) || {}
+            a = actual.dig(i, *nesting) || {}
+            ignore_default(e, a, defaults)
+          end
+        end
+
+        # expand nested widgets into expected/actual pairs for default resolution
+        # [a, e] -> [[a-w, e-w], [a-w1-w1, e-w1-w1], ...]
+        def widgets_pairs(*pair)
+          result = [pair.map { |d| d[:widgets] || [] }]
+          slots = result[0].map(&:size).max
+          slots.times do |i|
+            nested = pair.map { |d| d.dig(:widgets, i, :definition, :widgets) || [] }
+            result << nested if nested.any?(&:any?)
+          end
+          result
+        end
+      end
+
+      def as_json
+        return @json if @json
+        all_widgets = render_definitions + widgets
+        expand_q all_widgets
+
+        @json = {
+          layout_type: layout_type,
+          title: "#{title}#{LOCK}",
+          description: description,
+          template_variables: render_template_variables,
+          template_variable_presets: template_variable_presets,
+          widgets: all_widgets
+        }
+
+        @json[:id] = id if id
+
+        validate_json(@json) if validate
+
+        @json
+      end
+
+      def url(id)
+        Utils.path_to_url "/dashboard/#{id}"
+      end
+
+      def self.parse_url(url)
+        url[/\/dashboard\/([a-z\d-]+)/, 1]
+      end
+
+      def resolve_linked_tracking_ids!(id_map, **args)
+        widgets = as_json[:widgets].flat_map { |w| [w, *w.dig(:definition, :widgets) || []] }
+        widgets.each do |widget|
+          next unless definition = widget[:definition]
+          case definition[:type]
+          when "uptime"
+            if ids = definition[:monitor_ids]
+              definition[:monitor_ids] = ids.map do |id|
+                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+              end
+            end
+          when "alert_graph"
+            if (id = definition[:alert_id]) && tracking_id?(id)
+              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+            end
+          when "slo"
+            if (id = definition[:slo_id]) && tracking_id?(id)
+              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+            end
+          end
+        end
+      end
+
+      private
+
+      def tracking_id?(id)
+        id.is_a?(String) && id.include?(":")
+      end
+
+      # creates queries from metadata to avoid having to keep q and expression in sync
+      #
+      # {q: :metadata, metadata: [{expression: "sum:bar", alias_name: "foo"}, ...], }
+      # -> {q: "sum:bar, ...", metadata: ..., }
+      def expand_q(widgets)
+        widgets = widgets.flat_map { |w| w.dig(:definition, :widgets) || w } # expand groups
+        widgets.each do |w|
+          w.dig(:definition, :requests)&.each do |request|
+            next unless request.is_a?(Hash) && request[:q] == :metadata
+            request[:q] = request.fetch(:metadata).map { |m| m.fetch(:expression) }.join(", ")
+          end
+        end
+      end
+
+      def validate_json(data)
+        super
+
+        validate_template_variables data, :widgets
+
+        # Avoid diff from datadog presets sorting.
+        presets = data[:template_variable_presets]
+        invalid! "template_variable_presets must be sorted by name" if presets && presets != presets.sort_by { |p| p[:name] }
+      end
+
+      def render_definitions
+        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
+          # validate inputs
+          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
+            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+          end
+          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+          end
+
+          # build definition
+          requests = Array(queries).map do |q|
+            request = { q: q }
+            request[:display_type] = display_type if display_type
+            request
+          end
+          { definition: { title: title, type: type, requests: requests, **options } }
+        end
+      end
+    end
+  end
+end

data/lib/kennel/models/monitor.rb

@@ -0,0 +1,213 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Monitor < Record
+      include OptionalValidations
+
+      RENOTIFY_INTERVALS = [0, 10, 20, 30, 40, 50, 60, 90, 120, 180, 240, 300, 360, 720, 1440].freeze # minutes
+      # 2d and 1w are valid timeframes for anomaly monitors
+      QUERY_INTERVALS = ["1m", "5m", "10m", "15m", "30m", "1h", "2h", "4h", "1d", "2d", "1w"].freeze
+      OPTIONAL_SERVICE_CHECK_THRESHOLDS = [:ok, :warning].freeze
+      READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
+        :multi, :matching_downtimes, :overall_state_modified, :overall_state, :restricted_roles
+      ]
+
+      # defaults that datadog uses when options are not sent, so safe to leave out if our values match their defaults
+      MONITOR_OPTION_DEFAULTS = {
+        evaluation_delay: nil,
+        new_host_delay: 300,
+        timeout_h: 0,
+        renotify_interval: 0,
+        no_data_timeframe: nil # this works out ok since if notify_no_data is on, it would never be nil
+      }.freeze
+      DEFAULT_ESCALATION_MESSAGE = ["", nil].freeze
+
+      settings(
+        :query, :name, :message, :escalation_message, :critical, :type, :renotify_interval, :warning, :timeout_h, :evaluation_delay,
+        :ok, :no_data_timeframe, :notify_no_data, :notify_audit, :tags, :critical_recovery, :warning_recovery, :require_full_window,
+        :threshold_windows, :new_host_delay
+      )
+
+      defaults(
+        message: -> { "\n\n#{project.mention}" },
+        escalation_message: -> { DEFAULT_ESCALATION_MESSAGE.first },
+        renotify_interval: -> { project.team.renotify_interval },
+        warning: -> { nil },
+        ok: -> { nil },
+        id: -> { nil },
+        notify_no_data: -> { true },
+        no_data_timeframe: -> { 60 },
+        notify_audit: -> { true },
+        new_host_delay: -> { MONITOR_OPTION_DEFAULTS.fetch(:new_host_delay) },
+        tags: -> { @project.tags },
+        timeout_h: -> { MONITOR_OPTION_DEFAULTS.fetch(:timeout_h) },
+        evaluation_delay: -> { MONITOR_OPTION_DEFAULTS.fetch(:evaluation_delay) },
+        critical_recovery: -> { nil },
+        warning_recovery: -> { nil },
+        threshold_windows: -> { nil }
+      )
+
+      def as_json
+        return @as_json if @as_json
+        data = {
+          name: "#{name}#{LOCK}",
+          type: type,
+          query: query.strip,
+          message: message.strip,
+          tags: tags.uniq,
+          options: {
+            timeout_h: timeout_h,
+            notify_no_data: notify_no_data,
+            no_data_timeframe: notify_no_data ? no_data_timeframe : nil,
+            notify_audit: notify_audit,
+            require_full_window: require_full_window,
+            new_host_delay: new_host_delay,
+            include_tags: true,
+            escalation_message: Utils.presence(escalation_message.strip),
+            evaluation_delay: evaluation_delay,
+            locked: false, # setting this to true prevents any edit and breaks updates when using replace workflow
+            renotify_interval: renotify_interval || 0
+          }
+        }
+
+        data[:id] = id if id
+
+        options = data[:options]
+        if data.fetch(:type) != "composite"
+          thresholds = (options[:thresholds] = { critical: critical })
+
+          # warning, ok, critical_recovery, and warning_recovery are optional
+          [:warning, :ok, :critical_recovery, :warning_recovery].each do |key|
+            if value = send(key)
+              thresholds[key] = value
+            end
+          end
+
+          thresholds[:critical] = critical unless
+          case data.fetch(:type)
+          when "service check"
+            # avoid diff for default values of 1
+            OPTIONAL_SERVICE_CHECK_THRESHOLDS.each { |t| thresholds[t] ||= 1 }
+          when "query alert"
+            # metric and query values are stored as float by datadog
+            thresholds.each { |k, v| thresholds[k] = Float(v) }
+          end
+        end
+
+        if windows = threshold_windows
+          options[:threshold_windows] = windows
+        end
+
+        validate_json(data) if validate
+
+        @as_json = data
+      end
+
+      def resolve_linked_tracking_ids!(id_map, **args)
+        if as_json[:type] == "composite"
+          as_json[:query] = as_json[:query].gsub(/%\{(.*?)\}/) do
+            resolve_link($1, :monitor, id_map, **args)
+          end
+        end
+      end
+
+      def self.api_resource
+        "monitor"
+      end
+
+      def url(id)
+        Utils.path_to_url "/monitors##{id}/edit"
+      end
+
+      # datadog uses both / and # as separator in it's links
+      def self.parse_url(url)
+        return unless id = url[/\/monitors[\/#](\d+)/, 1]
+        Integer(id)
+      end
+
+      def self.normalize(expected, actual)
+        super
+        options = actual.fetch(:options)
+        options.delete(:silenced) # we do not manage silenced, so ignore it when diffing
+
+        # fields are not returned when set to true
+        if ["service check", "event alert"].include?(actual[:type])
+          options[:include_tags] = true unless options.key?(:include_tags)
+          options[:require_full_window] = true unless options.key?(:require_full_window)
+        end
+
+        case actual[:type]
+        when "event alert"
+          # setting nothing results in thresholds not getting returned from the api
+          options[:thresholds] ||= { critical: 0 }
+
+        when "service check"
+          # fields are not returned when created with default values via UI
+          OPTIONAL_SERVICE_CHECK_THRESHOLDS.each do |t|
+            options[:thresholds][t] ||= 1
+          end
+        end
+
+        # nil / "" / 0 are not returned from the api when set via the UI
+        options[:evaluation_delay] ||= nil
+
+        expected_options = expected[:options] || {}
+        ignore_default(expected_options, options, MONITOR_OPTION_DEFAULTS)
+        if DEFAULT_ESCALATION_MESSAGE.include?(options[:escalation_message])
+          options.delete(:escalation_message)
+          expected_options.delete(:escalation_message)
+        end
+      end
+
+      private
+
+      def require_full_window
+        # default 'on_average', 'at_all_times', 'in_total' aggregations to true, otherwise false
+        # https://docs.datadoghq.com/ap/#create-a-monitor
+        type != "query alert" || query.start_with?("avg", "min", "sum")
+      end
+
+      def validate_json(data)
+        super
+
+        type = data.fetch(:type)
+
+        # do not allow deprecated type that will be coverted by datadog and then produce a diff
+        if type == "metric alert"
+          invalid! "type 'metric alert' is deprecated, please set to a different type (e.g. 'query alert')"
+        end
+
+        # verify query includes critical value
+        if query_value = data.fetch(:query)[/\s*[<>]=?\s*(\d+(\.\d+)?)\s*$/, 1]
+          if Float(query_value) != Float(data.dig(:options, :thresholds, :critical))
+            invalid! "critical and value used in query must match"
+          end
+        end
+
+        # verify renotify interval is valid
+        unless RENOTIFY_INTERVALS.include? data.dig(:options, :renotify_interval)
+          invalid! "renotify_interval must be one of #{RENOTIFY_INTERVALS.join(", ")}"
+        end
+
+        if type == "query alert"
+          # verify interval is valud
+          interval = data.fetch(:query)[/\(last_(\S+?)\)/, 1]
+          if interval && !QUERY_INTERVALS.include?(interval)
+            invalid! "query interval was #{interval}, but must be one of #{QUERY_INTERVALS.join(", ")}"
+          end
+        end
+
+        if ["query alert", "service check"].include?(type) # TODO: most likely more types need this
+          # verify is_match uses available variables
+          message = data.fetch(:message)
+          used = message.scan(/{{\s*#is_match\s*"([a-zA-Z\d_.-]+).name"/).flatten.uniq
+          allowed = data.fetch(:query)[/by\s*[({]([^})]+)[})]/, 1].to_s.gsub(/["']/, "").split(/\s*,\s*/)
+          unsupported = used - allowed
+          if unsupported.any?
+            invalid! "is_match used with #{unsupported}, but metric is only grouped by #{allowed}"
+          end
+        end
+      end
+    end
+  end
+end