kennel 1.75.0

data/lib/kennel/file_cache.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+# cache that reads everything from a single file
+# to avoid doing multiple disk reads while iterating all definitions
+# it also replaces updated keys and has an overall expiry to not keep deleted things forever
+module Kennel
+  class FileCache
+    def initialize(file, cache_version)
+      @file = file
+      @cache_version = cache_version
+      @now = Time.now.to_i
+      @expires = @now + (30 * 24 * 60 * 60) # 1 month
+    end
+
+    def open
+      load_data
+      expire_old_data
+      yield self
+    ensure
+      persist
+    end
+
+    def fetch(key, key_version)
+      old_value, old_version = @data[key]
+      return old_value if old_version == [key_version, @cache_version]
+
+      new_value = yield
+      @data[key] = [new_value, [key_version, @cache_version], @expires]
+      new_value
+    end
+
+    private
+
+    def load_data
+      @data =
+        begin
+          Marshal.load(File.read(@file)) # rubocop:disable Security/MarshalLoad
+        rescue StandardError
+          {}
+        end
+    end
+
+    def persist
+      dir = File.dirname(@file)
+      FileUtils.mkdir_p(dir) unless File.directory?(dir)
+      File.write(@file, Marshal.dump(@data))
+    end
+
+    def expire_old_data
+      @data.reject! { |_, (_, _, ex)| ex < @now }
+    end
+  end
+end

data/lib/kennel/github_reporter.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+# Not used in here, but in our templated repo ... so keeping it around for now.
+module Kennel
+  class GithubReporter
+    MAX_COMMENT_SIZE = 65536
+    TRUNCATED_MSG = "\n```\n... (truncated)" # finish the code block so it look nice
+
+    class << self
+      def report(token, &block)
+        return yield unless token
+        new(token, Utils.capture_sh("git rev-parse HEAD").strip).report(&block)
+      end
+    end
+
+    def initialize(token, git_sha)
+      @token = token
+      @git_sha = git_sha
+      origin = ENV["PROJECT_REPOSITORY"] || Utils.capture_sh("git remote -v").split("\n").first
+      @repo_part = origin[%r{github\.com[:/](.+?)(\.git|$)}, 1] || raise("no origin found")
+    end
+
+    def report(&block)
+      output = Utils.strip_shell_control(Utils.tee_output(&block).strip)
+    rescue StandardError
+      output = "Error:\n#{$ERROR_INFO.message}"
+      raise
+    ensure
+      comment "```\n#{output || "Error"}\n```"
+    end
+
+    # https://developer.github.com/v3/repos/comments/#create-a-commit-comment
+    def comment(body)
+      # truncate to maximum allowed comment size for github to avoid 422
+      if body.bytesize > MAX_COMMENT_SIZE
+        body = body.byteslice(0, MAX_COMMENT_SIZE - TRUNCATED_MSG.bytesize) + TRUNCATED_MSG
+      end
+
+      post "commits/#{@git_sha}/comments", body: body
+    end
+
+    private
+
+    def post(path, data)
+      url = "https://api.github.com/repos/#{@repo_part}/#{path}"
+      response = Faraday.post(url, data.to_json, authorization: "token #{@token}")
+      raise "failed to POST to github:\n#{url} -> #{response.status}\n#{response.body}" unless response.status == 201
+    end
+  end
+end

data/lib/kennel/importer.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+module Kennel
+  class Importer
+    TITLES = [:name, :title].freeze
+    SORT_ORDER = [*TITLES, :id, :kennel_id, :type, :tags, :query, *Syncer::TRACKING_FIELDS, :template_variables].freeze
+
+    def initialize(api)
+      @api = api
+    end
+
+    def import(resource, id)
+      if ["screen", "dash"].include?(resource)
+        raise ArgumentError, "resource 'screen' and 'dash' are deprecated, use 'dashboard'"
+      end
+
+      model =
+        begin
+          Kennel::Models.const_get(resource.capitalize)
+        rescue NameError
+          raise ArgumentError, "#{resource} is not supported"
+        end
+
+      data = @api.show(model.api_resource, id)
+      id = data.fetch(:id) # keep native value
+      model.normalize({}, data) # removes id
+      data[:id] = id
+
+      title_field = TITLES.detect { |f| data[f] }
+      title = data.fetch(title_field)
+      title.tr!(Kennel::Models::Record::LOCK, "") # avoid double lock icon
+
+      # calculate or reuse kennel_id
+      # TODO: this is copy-pasted from syncer, need to find a nice way to reuse it
+      tracking_field = Syncer::TRACKING_FIELDS.detect { |f| data[f] }
+      data[:kennel_id] =
+        if tracking_field && data[tracking_field].sub!(/\n?-- Managed by kennel (\S+:\S+).*/, "")
+          $1.split(":").last
+        else
+          Kennel::Utils.parameterize(title)
+        end
+
+      case resource
+      when "monitor"
+        # flatten monitor options so they are all on the base
+        data.merge!(data.delete(:options))
+        data.merge!(data.delete(:thresholds) || {})
+        [:notify_no_data, :notify_audit].each { |k| data.delete(k) if data[k] } # monitor uses true by default
+        data = data.slice(*model.instance_methods)
+
+        # make query use critical method if it matches
+        critical = data[:critical]
+        query = data[:query]
+        if query && critical
+          query.sub!(/([><=]) (#{Regexp.escape(critical.to_f.to_s)}|#{Regexp.escape(critical.to_i.to_s)})$/, "\\1 \#{critical}")
+        end
+
+        data[:type] = "query alert" if data[:type] == "metric alert"
+      when "dashboard"
+        widgets = data[:widgets]&.flat_map { |widget| widget.dig(:definition, :widgets) || [widget] }
+        widgets&.each { |widget| dry_up_query!(widget) }
+      end
+
+      # simplify template_variables to array of string when possible
+      if vars = data[:template_variables]
+        vars.map! { |v| v[:default] == "*" && v[:prefix] == v[:name] ? v[:name] : v }
+      end
+
+      pretty = pretty_print(data).lstrip.gsub("\\#", "#")
+      <<~RUBY
+        #{model.name}.new(
+          self,
+          #{pretty}
+        )
+      RUBY
+    end
+
+    private
+
+    # reduce duplication in imports by using dry `q: :metadata` when possible
+    def dry_up_query!(widget)
+      (widget.dig(:definition, :requests) || []).each do |request|
+        next unless request.is_a?(Hash)
+        next unless metadata = request[:metadata]
+        next unless query = request[:q]&.dup
+        metadata.each do |m|
+          next unless exp = m[:expression]
+          query.sub!(exp, "")
+        end
+        request[:q] = :metadata if query.delete(", ") == ""
+      end
+    end
+
+    def pretty_print(hash)
+      sort_widgets hash
+
+      sort_hash(hash).map do |k, v|
+        pretty_value =
+          if v.is_a?(Hash) || (v.is_a?(Array) && !v.all? { |e| e.is_a?(String) })
+            # update answer here when changing https://stackoverflow.com/questions/8842546/best-way-to-pretty-print-a-hash
+            # (exclude last indent gsub)
+            pretty = JSON.pretty_generate(v)
+              .gsub(": null", ": nil")
+              .gsub(/(^\s*)"([a-zA-Z][a-zA-Z\d_]*)":/, "\\1\\2:") # "foo": 1 -> foo: 1
+              .gsub(/: \[\n\s+\]/, ": []") # empty arrays on a single line
+              .gsub(/^/, "    ") # indent
+              .gsub('q: "metadata"', "q: :metadata") # bring symbols back
+
+            "\n#{pretty}\n  "
+          elsif k == :message
+            "\n    <<~TEXT\n#{v.each_line.map { |l| l.strip.empty? ? "\n" : "      #{l}" }.join}\n      \#{super()}\n    TEXT\n  "
+          elsif k == :tags
+            " super() + #{v.inspect} "
+          else
+            " #{v.inspect} "
+          end
+        "  #{k}: -> {#{pretty_value}}"
+      end.join(",\n")
+    end
+
+    # sort dashboard widgets + nesting
+    def sort_widgets(outer)
+      outer[:widgets]&.each do |widgets|
+        definition = widgets[:definition]
+        definition.replace sort_hash(definition)
+        sort_widgets definition
+      end
+    end
+
+    # important to the front and rest deterministic
+    def sort_hash(hash)
+      Hash[hash.sort_by { |k, _| [SORT_ORDER.index(k) || 999, k] }]
+    end
+  end
+end

data/lib/kennel/models/base.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+require "hashdiff"
+
+module Kennel
+  module Models
+    class Base
+      extend SubclassTracking
+      include SettingsAsMethods
+
+      SETTING_OVERRIDABLE_METHODS = [:name, :kennel_id].freeze
+
+      def kennel_id
+        name = self.class.name
+        if name.start_with?("Kennel::")
+          raise_with_location ArgumentError, "Set :kennel_id"
+        end
+        @kennel_id ||= Utils.snake_case name
+      end
+
+      def name
+        self.class.name
+      end
+
+      def to_json # rubocop:disable Lint/ToJSON
+        raise NotImplementedError, "Use as_json"
+      end
+    end
+  end
+end

data/lib/kennel/models/dashboard.rb

@@ -0,0 +1,209 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Dashboard < Record
+      include TemplateVariables
+      include OptionalValidations
+
+      API_LIST_INCOMPLETE = true
+      DASHBOARD_DEFAULTS = { template_variables: [] }.freeze
+      READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [
+        :author_handle, :author_name, :modified_at, :url, :is_read_only, :notify_list
+      ]
+      REQUEST_DEFAULTS = {
+        style: { line_width: "normal", palette: "dog_classic", line_type: "solid" }
+      }.freeze
+      WIDGET_DEFAULTS = {
+        "timeseries" => { show_legend: false, legend_size: "0" },
+        "note" => { background_color: "white", font_size: "14", show_tick: false, tick_edge: "left", tick_pos: "50%", text_align: "left" }
+      }.freeze
+      SUPPORTED_DEFINITION_OPTIONS = [:events, :markers, :precision].freeze
+
+      DEFAULTS = {
+        template_variable_presets: nil
+      }.freeze
+
+      settings :title, :description, :definitions, :widgets, :layout_type, :template_variable_presets
+
+      defaults(
+        description: -> { "" },
+        definitions: -> { [] },
+        widgets: -> { [] },
+        template_variable_presets: -> { DEFAULTS.fetch(:template_variable_presets) },
+        id: -> { nil }
+      )
+
+      class << self
+        def api_resource
+          "dashboard"
+        end
+
+        def normalize(expected, actual)
+          super
+
+          ignore_default(expected, actual, DEFAULTS)
+
+          widgets_pairs(expected, actual).each do |pair|
+            # conditional_formats ordering is randomly changed by datadog, compare a stable ordering
+            pair.each do |widgets|
+              widgets.each do |widget|
+                if formats = widget.dig(:definition, :conditional_formats)
+                  widget[:definition][:conditional_formats] = formats.sort_by(&:hash)
+                end
+              end
+            end
+
+            ignore_widget_defaults pair
+
+            ignore_request_defaults(*pair)
+
+            # ids are kinda random so we always discard them
+            pair.each { |widgets| widgets.each { |w| w.delete(:id) } }
+          end
+        end
+
+        private
+
+        def ignore_widget_defaults(pair)
+          pair.map(&:size).max.times do |i|
+            types = pair.map { |w| w.dig(i, :definition, :type) }.uniq
+            next unless types.size == 1
+            next unless defaults = WIDGET_DEFAULTS[types.first]
+            ignore_defaults(pair[0], pair[1], defaults, nesting: :definition)
+          end
+        end
+
+        # discard styles/conditional_formats/aggregator if nothing would change when we applied (both are default or nil)
+        def ignore_request_defaults(expected, actual)
+          [expected.size, actual.size].max.times do |i|
+            a_r = actual.dig(i, :definition, :requests) || []
+            e_r = expected.dig(i, :definition, :requests) || []
+            ignore_defaults e_r, a_r, REQUEST_DEFAULTS
+          end
+        end
+
+        def ignore_defaults(expected, actual, defaults, nesting: nil)
+          [expected.size, actual.size].max.times do |i|
+            e = expected.dig(i, *nesting) || {}
+            a = actual.dig(i, *nesting) || {}
+            ignore_default(e, a, defaults)
+          end
+        end
+
+        # expand nested widgets into expected/actual pairs for default resolution
+        # [a, e] -> [[a-w, e-w], [a-w1-w1, e-w1-w1], ...]
+        def widgets_pairs(*pair)
+          result = [pair.map { |d| d[:widgets] || [] }]
+          slots = result[0].map(&:size).max
+          slots.times do |i|
+            nested = pair.map { |d| d.dig(:widgets, i, :definition, :widgets) || [] }
+            result << nested if nested.any?(&:any?)
+          end
+          result
+        end
+      end
+
+      def as_json
+        return @json if @json
+        all_widgets = render_definitions + widgets
+        expand_q all_widgets
+
+        @json = {
+          layout_type: layout_type,
+          title: "#{title}#{LOCK}",
+          description: description,
+          template_variables: render_template_variables,
+          template_variable_presets: template_variable_presets,
+          widgets: all_widgets
+        }
+
+        @json[:id] = id if id
+
+        validate_json(@json) if validate
+
+        @json
+      end
+
+      def url(id)
+        Utils.path_to_url "/dashboard/#{id}"
+      end
+
+      def self.parse_url(url)
+        url[/\/dashboard\/([a-z\d-]+)/, 1]
+      end
+
+      def resolve_linked_tracking_ids!(id_map, **args)
+        widgets = as_json[:widgets].flat_map { |w| [w, *w.dig(:definition, :widgets) || []] }
+        widgets.each do |widget|
+          next unless definition = widget[:definition]
+          case definition[:type]
+          when "uptime"
+            if ids = definition[:monitor_ids]
+              definition[:monitor_ids] = ids.map do |id|
+                tracking_id?(id) ? resolve_link(id, :monitor, id_map, **args) : id
+              end
+            end
+          when "alert_graph"
+            if (id = definition[:alert_id]) && tracking_id?(id)
+              definition[:alert_id] = resolve_link(id, :monitor, id_map, **args).to_s
+            end
+          when "slo"
+            if (id = definition[:slo_id]) && tracking_id?(id)
+              definition[:slo_id] = resolve_link(id, :slo, id_map, **args).to_s
+            end
+          end
+        end
+      end
+
+      private
+
+      def tracking_id?(id)
+        id.is_a?(String) && id.include?(":")
+      end
+
+      # creates queries from metadata to avoid having to keep q and expression in sync
+      #
+      # {q: :metadata, metadata: [{expression: "sum:bar", alias_name: "foo"}, ...], }
+      # -> {q: "sum:bar, ...", metadata: ..., }
+      def expand_q(widgets)
+        widgets = widgets.flat_map { |w| w.dig(:definition, :widgets) || w } # expand groups
+        widgets.each do |w|
+          w.dig(:definition, :requests)&.each do |request|
+            next unless request.is_a?(Hash) && request[:q] == :metadata
+            request[:q] = request.fetch(:metadata).map { |m| m.fetch(:expression) }.join(", ")
+          end
+        end
+      end
+
+      def validate_json(data)
+        super
+
+        validate_template_variables data, :widgets
+
+        # Avoid diff from datadog presets sorting.
+        presets = data[:template_variable_presets]
+        invalid! "template_variable_presets must be sorted by name" if presets && presets != presets.sort_by { |p| p[:name] }
+      end
+
+      def render_definitions
+        definitions.map do |title, type, display_type, queries, options = {}, ignored = nil|
+          # validate inputs
+          if ignored || (!title || !type || !queries || !options.is_a?(Hash))
+            raise ArgumentError, "Expected exactly 5 arguments for each definition (title, type, display_type, queries, options)"
+          end
+          if (SUPPORTED_DEFINITION_OPTIONS | options.keys) != SUPPORTED_DEFINITION_OPTIONS
+            raise ArgumentError, "Supported options are: #{SUPPORTED_DEFINITION_OPTIONS.map(&:inspect).join(", ")}"
+          end
+
+          # build definition
+          requests = Array(queries).map do |q|
+            request = { q: q }
+            request[:display_type] = display_type if display_type
+            request
+          end
+          { definition: { title: title, type: type, requests: requests, **options } }
+        end
+      end
+    end
+  end
+end