kennel 1.74.0

data/lib/kennel/models/project.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Project < Base
+      settings :team, :parts, :tags, :mention
+      defaults(
+        tags: -> { ["service:#{kennel_id}"] + team.tags },
+        mention: -> { team.mention }
+      )
+
+      def self.file_location
+        @file_location ||= begin
+          method_in_file = instance_methods(false).first
+          instance_method(method_in_file).source_location.first.sub("#{Bundler.root}/", "")
+        end
+      end
+
+      def validated_parts
+        all = parts
+        validate_parts(all)
+        all
+      end
+
+      private
+
+      # hook for users to add custom validations via `prepend`
+      def validate_parts(parts)
+      end
+    end
+  end
+end

data/lib/kennel/models/record.rb

@@ -0,0 +1,94 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Record < Base
+      LOCK = "\u{1F512}"
+      READONLY_ATTRIBUTES = [
+        :deleted, :id, :created, :created_at, :creator, :org_id, :modified, :modified_at, :api_resource
+      ].freeze
+      API_LIST_INCOMPLETE = false
+
+      settings :id, :kennel_id
+
+      class << self
+        def parse_any_url(url)
+          subclasses.detect do |s|
+            if id = s.parse_url(url)
+              break s.api_resource, id
+            end
+          end
+        end
+
+        private
+
+        def normalize(_expected, actual)
+          self::READONLY_ATTRIBUTES.each { |k| actual.delete k }
+        end
+
+        def ignore_default(expected, actual, defaults)
+          definitions = [actual, expected]
+          defaults.each do |key, default|
+            if definitions.all? { |r| !r.key?(key) || r[key] == default }
+              actual.delete(key)
+              expected.delete(key)
+            end
+          end
+        end
+      end
+
+      attr_reader :project
+
+      def initialize(project, *args)
+        raise ArgumentError, "First argument must be a project, not #{project.class}" unless project.is_a?(Project)
+        @project = project
+        super(*args)
+      end
+
+      def diff(actual)
+        expected = as_json
+        expected.delete(:id)
+
+        self.class.send(:normalize, expected, actual)
+
+        # strict: ignore Integer vs Float
+        # similarity: show diff when not 100% similar
+        # use_lcs: saner output
+        Hashdiff.diff(actual, expected, use_lcs: false, strict: false, similarity: 1)
+      end
+
+      def tracking_id
+        "#{project.kennel_id}:#{kennel_id}"
+      end
+
+      def resolve_linked_tracking_ids!(*)
+      end
+
+      private
+
+      def resolve_link(id, type, id_map, force:)
+        value = id_map[id]
+        if value == :new
+          if force
+            # TODO: remove the need for this by sorting monitors by missing resolutions
+            invalid! "#{id} needs to already exist, try again"
+          else
+            id # will be re-resolved by syncer after the linked object was created
+          end
+        elsif value
+          value
+        else
+          invalid! "Unable to find #{type} #{id} (does not exist and is not being created by the current run)"
+        end
+      end
+
+      # let users know which project/resource failed when something happens during diffing where the backtrace is hidden
+      def invalid!(message)
+        raise ValidationError, "#{tracking_id} #{message}"
+      end
+
+      def raise_with_location(error, message)
+        super error, "#{message} for project #{project.kennel_id}"
+      end
+    end
+  end
+end

data/lib/kennel/models/slo.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Slo < Record
+      READONLY_ATTRIBUTES = superclass::READONLY_ATTRIBUTES + [:type_id, :monitor_tags]
+      DEFAULTS = {
+        description: nil,
+        query: nil,
+        groups: nil,
+        monitor_ids: [],
+        thresholds: []
+      }.freeze
+
+      settings :type, :description, :thresholds, :query, :tags, :monitor_ids, :monitor_tags, :name, :groups
+
+      defaults(
+        id: -> { nil },
+        tags: -> { @project.tags },
+        query: -> { DEFAULTS.fetch(:query) },
+        description: -> { DEFAULTS.fetch(:description) },
+        monitor_ids: -> { DEFAULTS.fetch(:monitor_ids) },
+        thresholds: -> { DEFAULTS.fetch(:thresholds) },
+        groups: -> { DEFAULTS.fetch(:groups) }
+      )
+
+      def initialize(*)
+        super
+        if thresholds.any? { |t| t[:warning] && t[:warning].to_f <= t[:critical].to_f }
+          raise ValidationError, "Threshold warning must be greater-than critical value"
+        end
+      end
+
+      def as_json
+        return @as_json if @as_json
+        data = {
+          name: "#{name}#{LOCK}",
+          description: description,
+          thresholds: thresholds,
+          monitor_ids: monitor_ids,
+          tags: tags.uniq,
+          type: type
+        }
+
+        if v = query
+          data[:query] = v
+        end
+        if v = id
+          data[:id] = v
+        end
+        if v = groups
+          data[:groups] = v
+        end
+
+        @as_json = data
+      end
+
+      def self.api_resource
+        "slo"
+      end
+
+      def url(id)
+        Utils.path_to_url "/slo?slo_id=#{id}"
+      end
+
+      def self.parse_url(url)
+        url[/\/slo\?slo_id=([a-z\d]+)/, 1]
+      end
+
+      def resolve_linked_tracking_ids!(id_map, **args)
+        as_json[:monitor_ids] = as_json[:monitor_ids].map do |id|
+          id.is_a?(String) ? resolve_link(id, :monitor, id_map, **args) : id
+        end
+      end
+
+      def self.normalize(expected, actual)
+        super
+
+        # remove readonly values
+        actual[:thresholds]&.each do |threshold|
+          threshold.delete(:warning_display)
+          threshold.delete(:target_display)
+        end
+
+        # tags come in a semi-random order and order is never updated
+        expected[:tags]&.sort!
+        actual[:tags].sort!
+
+        ignore_default(expected, actual, DEFAULTS)
+      end
+    end
+  end
+end

data/lib/kennel/models/team.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+module Kennel
+  module Models
+    class Team < Base
+      settings :mention, :tags, :renotify_interval, :kennel_id
+      defaults(
+        tags: -> { ["team:#{kennel_id.sub(/^teams_/, "")}"] },
+        renotify_interval: -> { 0 }
+      )
+    end
+  end
+end

data/lib/kennel/optional_validations.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+module Kennel
+  module OptionalValidations
+    def self.included(base)
+      base.settings :validate
+      base.defaults(validate: -> { true })
+    end
+
+    private
+
+    def validate_json(data)
+      bad = Kennel::Utils.all_keys(data).grep_v(Symbol)
+      return if bad.empty?
+      invalid!(
+        "Only use Symbols as hash keys to avoid permanent diffs when updating.\n" \
+        "Change these keys to be symbols (usually 'foo' => 1 --> 'foo': 1)\n" \
+        "#{bad.map(&:inspect).join("\n")}"
+      )
+    end
+  end
+end

data/lib/kennel/progress.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+require "benchmark"
+
+module Kennel
+  class Progress
+    # print what we are doing and a spinner until it is done ... then show how long it took
+    def self.progress(name)
+      Kennel.err.print "#{name} ... "
+
+      animation = "-\\|/"
+      count = 0
+      stop = false
+      result = nil
+
+      spinner = Thread.new do
+        loop do
+          break if stop
+          Kennel.err.print animation[count % animation.size]
+          sleep 0.2
+          Kennel.err.print "\b"
+          count += 1
+        end
+      end
+
+      time = Benchmark.realtime { result = yield }
+
+      stop = true
+      spinner.join
+      Kennel.err.print "#{time.round(2)}s\n"
+
+      result
+    end
+  end
+end

data/lib/kennel/settings_as_methods.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+module Kennel
+  module SettingsAsMethods
+    SETTING_OVERRIDABLE_METHODS = [].freeze
+
+    def self.included(base)
+      base.extend ClassMethods
+      base.instance_variable_set(:@settings, [])
+    end
+
+    module ClassMethods
+      def settings(*names)
+        duplicates = (@settings & names)
+        if duplicates.any?
+          raise ArgumentError, "Settings #{duplicates.map(&:inspect).join(", ")} are already defined"
+        end
+
+        overrides = ((instance_methods - self::SETTING_OVERRIDABLE_METHODS) & names)
+        if overrides.any?
+          raise ArgumentError, "Settings #{overrides.map(&:inspect).join(", ")} are already used as methods"
+        end
+
+        @settings.concat names
+
+        names.each do |name|
+          next if method_defined?(name)
+          define_method name do
+            raise_with_location ArgumentError, "'#{name}' on #{self.class} was not set or passed as option"
+          end
+        end
+      end
+
+      def defaults(options)
+        options.each do |name, block|
+          validate_setting_exist name
+          define_method name, &block
+        end
+      end
+
+      private
+
+      def validate_setting_exist(name)
+        return if !@settings || @settings.include?(name)
+        supported = @settings.map(&:inspect)
+        raise ArgumentError, "Unsupported setting #{name.inspect}, supported settings are #{supported.join(", ")}"
+      end
+
+      def inherited(child)
+        super
+        child.instance_variable_set(:@settings, (@settings || []).dup)
+      end
+    end
+
+    def initialize(options = {})
+      super()
+
+      unless options.is_a?(Hash)
+        raise ArgumentError, "Expected #{self.class.name}.new options to be a Hash, got a #{options.class}"
+      end
+
+      options.each do |k, v|
+        next if v.class == Proc
+        raise ArgumentError, "Expected #{self.class.name}.new option :#{k} to be Proc, for example `#{k}: -> { 12 }`"
+      end
+
+      options.each do |name, block|
+        self.class.send :validate_setting_exist, name
+        define_singleton_method name, &block
+      end
+
+      # need expand_path so it works wih rake and when run individually
+      pwd = /^#{Regexp.escape(Dir.pwd)}\//
+      @invocation_location = caller.detect do |l|
+        if found = File.expand_path(l).sub!(pwd, "")
+          break found
+        end
+      end
+    end
+
+    def raise_with_location(error, message)
+      message = message.dup
+      message << " on #{@invocation_location}" if @invocation_location
+      raise error, message
+    end
+  end
+end

data/lib/kennel/subclass_tracking.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+module Kennel
+  module SubclassTracking
+    def recursive_subclasses
+      subclasses + subclasses.flat_map(&:recursive_subclasses)
+    end
+
+    def subclasses
+      @subclasses ||= []
+    end
+
+    private
+
+    def inherited(child)
+      super
+      subclasses << child
+    end
+  end
+end

data/lib/kennel/syncer.rb

@@ -0,0 +1,260 @@
+# frozen_string_literal: true
+module Kennel
+  class Syncer
+    CACHE_FILE = "tmp/cache/details" # keep in sync with .travis.yml caching
+    TRACKING_FIELDS = [:message, :description].freeze
+    DELETE_ORDER = ["dashboard", "slo", "monitor"].freeze # dashboards references monitors + slos, slos reference monitors
+
+    def initialize(api, expected, project: nil)
+      @api = api
+      @project_filter = project
+      @expected = expected
+      if @project_filter
+        original = @expected
+        @expected = @expected.select { |e| e.project.kennel_id == @project_filter }
+        if @expected.empty?
+          possible = original.map { |e| e.project.kennel_id }.uniq.sort
+          raise "#{@project_filter} does not match any projects, try any of these:\n#{possible.join("\n")}"
+        end
+      end
+      @expected.each { |e| add_tracking_id e }
+      calculate_diff
+      prevent_irreversible_partial_updates
+    end
+
+    def plan
+      Kennel.out.puts "Plan:"
+      if noop?
+        Kennel.out.puts Utils.color(:green, "Nothing to do")
+      else
+        print_plan "Create", @create, :green
+        print_plan "Update", @update, :yellow
+        print_plan "Delete", @delete, :red
+      end
+    end
+
+    def confirm
+      ENV["CI"] || !STDIN.tty? || Utils.ask("Execute Plan ?") unless noop?
+    end
+
+    def update
+      changed = (@create + @update).map { |_, e| e } unless @create.empty?
+
+      @create.each do |_, e|
+        e.resolve_linked_tracking_ids!({}, force: true)
+
+        reply = @api.create e.class.api_resource, e.as_json
+        id = reply.fetch(:id)
+
+        # resolve ids we could previously no resolve
+        changed.delete e
+        resolve_linked_tracking_ids! from: [reply], to: changed
+
+        Kennel.out.puts "Created #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+      end
+
+      @update.each do |id, e|
+        e.resolve_linked_tracking_ids!({}, force: true)
+        @api.update e.class.api_resource, id, e.as_json
+        Kennel.out.puts "Updated #{e.class.api_resource} #{tracking_id(e.as_json)} #{e.url(id)}"
+      end
+
+      @delete.each do |id, _, a|
+        @api.delete a.fetch(:api_resource), id
+        Kennel.out.puts "Deleted #{a.fetch(:api_resource)} #{tracking_id(a)} #{id}"
+      end
+    end
+
+    private
+
+    def noop?
+      @create.empty? && @update.empty? && @delete.empty?
+    end
+
+    def calculate_diff
+      @update = []
+      @delete = []
+
+      actual = Progress.progress("Downloading definitions") { download_definitions }
+      resolve_linked_tracking_ids! from: actual, to: @expected
+      filter_by_project! actual
+
+      Progress.progress "Diffing" do
+        items = actual.map do |a|
+          e = matching_expected(a)
+          if e && @expected.delete(e)
+            [e, a]
+          else
+            [nil, a]
+          end
+        end
+
+        details_cache do |cache|
+          # fill details of things we need to compare (only do this part in parallel for safety & balancing)
+          Utils.parallel(items.select { |e, _| e && e.class::API_LIST_INCOMPLETE }) { |_, a| fill_details(a, cache) }
+        end
+
+        # pick out things to update or delete
+        items.each do |e, a|
+          id = a.fetch(:id)
+          if e
+            diff = e.diff(a)
+            @update << [id, e, a, diff] if diff.any?
+          elsif tracking_id(a) # was previously managed
+            @delete << [id, nil, a]
+          end
+        end
+
+        ensure_all_ids_found
+        @create = @expected.map { |e| [nil, e] }
+        @create.sort_by! { |_, e| -DELETE_ORDER.index(e.class.api_resource) }
+      end
+
+      @delete.sort_by! { |_, _, a| DELETE_ORDER.index a.fetch(:api_resource) }
+    end
+
+    # Make diff work even though we cannot mass-fetch definitions
+    def fill_details(a, cache)
+      resource = a.fetch(:api_resource)
+      args = [resource, a.fetch(:id)]
+      full = cache.fetch(args, a[:modified] || a.fetch(:modified_at)) do
+        @api.show(*args)
+      end
+      a.merge!(full)
+    end
+
+    def details_cache(&block)
+      cache = FileCache.new CACHE_FILE, Kennel::VERSION
+      cache.open(&block)
+    end
+
+    def download_definitions
+      Utils.parallel(Models::Record.subclasses.map(&:api_resource)) do |api_resource|
+        results = @api.list(api_resource, with_downtimes: false) # lookup monitors without adding unnecessary downtime information
+        results = results[results.keys.first] if results.is_a?(Hash) # dashboards are nested in {dashboards: []}
+        results.each { |c| c[:api_resource] = api_resource } # store api resource for later diffing
+      end.flatten(1)
+    end
+
+    def ensure_all_ids_found
+      @expected.each do |e|
+        next unless id = e.id
+        raise "Unable to find existing #{e.class.api_resource} with id #{id}"
+      end
+    end
+
+    def matching_expected(a)
+      # index list by all the thing we look up by: tracking id and actual id
+      @lookup_map ||= @expected.each_with_object({}) do |e, all|
+        keys = [tracking_id(e.as_json)]
+        keys << "#{e.class.api_resource}:#{e.id}" if e.id
+        keys.compact.each do |key|
+          raise "Lookup #{key} is duplicated" if all[key]
+          all[key] = e
+        end
+      end
+
+      @lookup_map["#{a.fetch(:api_resource)}:#{a.fetch(:id)}"] || @lookup_map[tracking_id(a)]
+    end
+
+    def print_plan(step, list, color)
+      return if list.empty?
+      list.each do |_, e, a, diff|
+        api_resource = (e ? e.class.api_resource : a.fetch(:api_resource))
+        Kennel.out.puts Utils.color(color, "#{step} #{api_resource} #{e&.tracking_id || tracking_id(a)}")
+        print_diff(diff) if diff # only for update
+      end
+    end
+
+    def print_diff(diff)
+      diff.each do |type, field, old, new|
+        if type == "+"
+          temp = Utils.pretty_inspect(new)
+          new = Utils.pretty_inspect(old)
+          old = temp
+        else # ~ and -
+          old = Utils.pretty_inspect(old)
+          new = Utils.pretty_inspect(new)
+        end
+
+        if (old + new).size > 100
+          Kennel.out.puts "  #{type}#{field}"
+          Kennel.out.puts "    #{old} ->"
+          Kennel.out.puts "    #{new}"
+        else
+          Kennel.out.puts "  #{type}#{field} #{old} -> #{new}"
+        end
+      end
+    end
+
+    # Do not add tracking-id when working with existing ids on a branch,
+    # so resource do not get deleted fr:om merges to master.
+    # Also make sure the diff still makes sense, by kicking out the now noop-update.
+    #
+    # Note: ideally we'd never add tracking in the first place, but at that point we do not know the diff yet
+    def prevent_irreversible_partial_updates
+      return unless @project_filter
+      @update.select! do |_, e, _, diff|
+        next true unless e.id # short circuit for performance
+
+        diff.select! do |field_diff|
+          (_, field, old, new) = field_diff
+          next true unless tracking_field?(field)
+
+          if (old_tracking = tracking_value(old))
+            old_tracking == tracking_value(new) || raise("do not update! (atm unreachable)")
+          else
+            field_diff[3] = remove_tracking_id(e) # make plan output match update
+            old != field_diff[3]
+          end
+        end
+
+        !diff.empty?
+      end
+    end
+
+    def resolve_linked_tracking_ids!(from:, to:)
+      map = from.each_with_object({}) { |a, lookup| lookup[tracking_id(a)] = a.fetch(:id) }
+      to.each { |e| map[e.tracking_id] ||= :new }
+      to.each { |e| e.resolve_linked_tracking_ids!(map, force: false) }
+    end
+
+    def filter_by_project!(definitions)
+      return unless @project_filter
+      definitions.select! do |a|
+        id = tracking_id(a)
+        !id || id.start_with?("#{@project_filter}:")
+      end
+    end
+
+    def add_tracking_id(e)
+      json = e.as_json
+      field = tracking_field(json)
+      raise "remove \"-- Managed by kennel\" line it from #{field} to copy a resource" if tracking_value(json[field])
+      json[field] = "#{json[field]}\n-- Managed by kennel #{e.tracking_id} in #{e.project.class.file_location}, do not modify manually".lstrip
+    end
+
+    def remove_tracking_id(e)
+      json = e.as_json
+      field = tracking_field(json)
+      value = json[field]
+      json[field] = value.dup.sub!(/\n?-- Managed by kennel .*/, "") || raise("did not find tracking id in #{value}")
+    end
+
+    def tracking_id(a)
+      tracking_value a[tracking_field(a)]
+    end
+
+    def tracking_value(content)
+      content.to_s[/-- Managed by kennel (\S+:\S+)/, 1]
+    end
+
+    def tracking_field(a)
+      TRACKING_FIELDS.detect { |f| a.key?(f) }
+    end
+
+    def tracking_field?(field)
+      TRACKING_FIELDS.include?(field.to_sym)
+    end
+  end
+end