codeclimate-collector-manager 0.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: a12a56ef4ce5f3dcec9de3baa06a263ed8bcf7e759114d9b70aa6ef2776b4e0b
+  data.tar.gz: a2a4cc6d0c504e3b1e544f3e5964a39865e7c71e5927df9aee60211d420141a3
+SHA512:
+  metadata.gz: 9464ecc990f434fac802f5ae75d8626dfcb0e68928632188c74d779fdd584a409ed2f3244ef5032e32b2a2a083d91562035c7db6cab2a2421dfbbd69f629424d
+  data.tar.gz: 24940ac19c4e2178d0ffa0dea265d8bc1351f41fab475a97157c45a11bab57a8836785b7c112dd9207c90bd367ff12f83ffef10645cdc2c1badba80d4b29aea7

data/bin/run-collector

@@ -0,0 +1,94 @@
+#!/usr/bin/env ruby
+#
+# Run a collector and log what it does for testing.
+#
+# Usage: bundle exec run-collector collector-slug collector-method config-file
+#
+# This script will `require "codeclimate-collector-#{collector-slug}"`,
+# instantiate a client (class name derived from the require name), providing a
+# configuration object constructed from the contents of `config-file`. It will
+# then ask the client to handle the request parsed from `request-file`.
+#
+# E.g. `bundle exec run-collector pagerduty sync config.json` will
+# `require "codeclimate-collector-pagerduty"`, construct a config from
+# `config.json`, and call
+# `Codeclimate::Collectors::Pagerduty::Client.sync(
+#   config,
+#   earliest_date_cutoff: 6.months.ago)`.
+
+require "codeclimate-collector-manager"
+
+class Runner
+  class MessagesHandler
+    def initialize(logger)
+      @logger = logger
+    end
+
+    def send_message(message)
+      logger.info("[MESSAGE RECEIVED] #{message.to_json}")
+    end
+
+    private
+
+    attr_reader :logger
+  end
+
+  def initialize(slug: , client_method:, config_path:)
+    require "codeclimate-collector-#{slug}"
+    @client_klass = "Codeclimate::Collectors::#{slug.camelcase}::Client".constantize
+    @client_method = client_method
+    @client_config = build_config(JSON.parse(File.read(config_path)))
+  end
+
+  def run
+    case client_method
+    when "sync"
+      client_klass.sync(
+        configuration: client_config,
+        manager: manager,
+        earliest_data_cutoff: Time.now.utc - (6 * 30 * 24 * 60 * 60), # ~6 months
+      )
+    when "validate_configuration"
+      client_klass.validate_configuration(
+        configuration: client_config,
+        manager: manager,
+      )
+    else
+      raise ArgumentError, "don't know how to process '#{client_method}'"
+    end
+
+    logger.info("Done!")
+  end
+
+  private
+
+  attr_reader :client_klass, :client_method, :client_config
+
+  def build_client
+    client_klass.new(configuration: client_config.dup, manager: manager)
+  end
+
+  def build_config(json)
+    Codeclimate::Collectors::Configuration.new(json)
+  end
+
+  def manager
+    @manager ||= Codeclimate::Collectors::Manager.new(
+      messages: Codeclimate::Collectors::MessagesFacade.new(
+        implementation: MessagesHandler.new(logger),
+      ),
+    )
+  end
+
+  def logger
+    @logger ||= Logger.new($stdout).tap do |l|
+      l.level = Logger::INFO
+    end
+  end
+end
+
+Runner.new(
+  slug: ARGV[0],
+  client_method: ARGV[1],
+  config_path: ARGV[2],
+).run

data/lib/codeclimate-collector-manager.rb

@@ -0,0 +1,4 @@
+require "active_model"
+require "active_support/json"
+
+require "codeclimate/collectors"

data/lib/codeclimate/collectors.rb

@@ -0,0 +1,5 @@
+require "codeclimate/collectors/configuration"
+require "codeclimate/collectors/manager"
+require "codeclimate/collectors/validations/type_validator"
+require "codeclimate/collectors/messages"
+require "codeclimate/collectors/messages_facade"

data/lib/codeclimate/collectors/configuration.rb

@@ -0,0 +1,34 @@
+module Codeclimate
+  module Collectors
+    # The configuration for a collector client.
+    # Used like a +Hash+. Treats all keys as symbols.
+    class Configuration
+      def initialize(attrs = {})
+        @storage = attrs.deep_symbolize_keys
+      end
+
+      def [](key)
+        storage[key.to_sym]
+      end
+
+      def []=(key, val)
+        storage[key.to_sym] = val
+      end
+
+      def fetch(*args)
+        case args.count
+        when 1
+          storage.fetch(args[0].to_sym)
+        when 2
+          storage.fetch(args[0].to_sym, args[1])
+        else
+          raise ArgumentError, "expected 1 or 2 arguments"
+        end
+      end
+
+      private
+
+      attr_reader :storage
+    end
+  end
+end

data/lib/codeclimate/collectors/manager.rb

@@ -0,0 +1,14 @@
+module Codeclimate
+  module Collectors
+    class Manager
+      VERSION = File.read(File.expand_path("../../../../VERSION", __FILE__)).strip
+
+      attr_reader :messages
+
+      # - +messages+ is an instance of MessagesFacade
+      def initialize(messages:)
+        @messages = messages
+      end
+    end
+  end
+end

data/lib/codeclimate/collectors/messages.rb

@@ -0,0 +1,12 @@
+require "codeclimate/collectors/messages/message"
+
+require "codeclimate/collectors/messages/configuration_verification"
+require "codeclimate/collectors/messages/incident"
+
+module Codeclimate
+  module Collectors
+    module Messages
+      InvalidMessage = Class.new(StandardError)
+    end
+  end
+end

data/lib/codeclimate/collectors/messages/configuration_verification.rb

@@ -0,0 +1,38 @@
+module Codeclimate
+  module Collectors
+    module Messages
+      # Used to emit the result of processing a +Requests::VerifyConfiguration+
+      # request
+      class ConfigurationVerification < Message
+        STATES = [
+          SUCCESS = "success".freeze,
+          ERROR = "error".freeze,
+        ].freeze
+
+        attribute :state, String
+        attribute :error_messages, Array, optional: true
+
+        validates :state, inclusion: { in: STATES }
+        validate :validate_error_messages_match_state
+        validate :validate_error_messages_are_strings
+
+        private
+
+        def validate_error_messages_are_strings
+          all_strs = (error_messages || []).all? { |m| m.is_a?(String) }
+          unless all_strs
+            errors.add(:error_message, "must be an array of strings")
+          end
+        end
+
+        def validate_error_messages_match_state
+          if state == ERROR && error_messages.blank?
+            errors.add(:error_messages, "are required if the state is 'error'")
+          elsif state == SUCCESS && error_messages.present?
+            errors.add(:error_messages, "should not be provided if config is valid")
+          end
+        end
+      end
+    end
+  end
+end

data/lib/codeclimate/collectors/messages/incident.rb

@@ -0,0 +1,41 @@
+module Codeclimate
+  module Collectors
+    module Messages
+      # Represents an incident from an Incident Response Platform, such as
+      # PagerDuty.
+      class Incident < Message
+        STATUSES = [
+          TRIGGERED = "triggered".freeze,
+          ACKNOWLEDGED = "acknowledged".freeze,
+          RESOLVED = "resolved".freeze,
+        ].freeze
+
+        # The unique identifier of this incident within the IRP. Often this is
+        # an +id+ key in API responses.
+        attribute :external_id, String
+
+        # The human-readable title of the incident.
+        attribute :title, String
+
+        # The URL a human would visit in a brower to see details of the
+        # incident.
+        attribute :url, String
+
+        # The incrementing numerical number of the incident.
+        attribute :number, Integer
+
+        # The current status of the incident.
+        attribute :status, String
+
+        # The time the incident began.
+        attribute :created_at, Time
+
+        validates :status, inclusion: { in: STATUSES }
+
+        def self.ordering_keys
+          [:external_id]
+        end
+      end
+    end
+  end
+end

data/lib/codeclimate/collectors/messages/message.rb

@@ -0,0 +1,91 @@
+module Codeclimate
+  module Collectors
+    module Messages
+      class Message
+        include ActiveModel::Model
+        include ActiveModel::Validations::HelperMethods
+        include Collectors::Validations
+
+        def self.attribute_metadata
+          @attribute_metadata ||= {}
+        end
+
+        # Declare an attribute on a message type. This will declare an
+        # +attr_accessor+, a validation that the attribute is of the specified
+        # +type+, and include it in +#attributes+. If +optional+ is false a
+        # validation for +presence+ is also added.
+        def self.attribute(name, type, optional: false)
+          name = name.to_sym
+
+          attribute_metadata[name] = { type: type, optional: optional }
+
+          attr_accessor name
+
+          validates name, presence: !optional, type: { type: type }
+        end
+
+        # Calculate the type string to use in the JSON representation to
+        # identify this message type.
+        def self.json_type
+          to_s.gsub(/^.+::(\w+)$/, "\\1").underscore
+        end
+
+        # Messages can have ordering dependencies, e.g. a collector could emit
+        # an Incident message, followed by several IncidentEvent messages.
+        # IncidentEvent messages identify their associated incident via their
+        # +incident_external_id+ attribute.
+        #
+        # Our backend distributes messages amongst parallel workers for
+        # ingestion. To ensure messages with an ordering dependency are
+        # processed in order by the same worker, we can return an appropriate
+        # +ordering_keys+ array for the classes: e.g. Incident returns
+        # +[:external_id]+ and IncidentEvent returns +[incident_external_id]+.
+        #
+        # Collectors are also responsible for emitting messages in the
+        # appropriate order.
+        def self.ordering_keys
+          []
+        end
+
+        def ==(other)
+          other.class == self.class && other.attributes == attributes
+        end
+
+        validate :validate_not_base_class
+
+        # Return all the attributes of a message as a +Hash+
+        def attributes
+          Hash[
+            self.class.attribute_metadata.keys.each.map do |name|
+              [name, public_send(name)]
+            end
+          ]
+        end
+
+        # A +Hash+ for serializing the message as JSON.
+        # Includes a +type+ key and an +attributes+ key.
+        def as_json(*_opts)
+          {
+            type: self.class.json_type,
+            attributes: json_attributes,
+          }
+        end
+
+        protected
+
+        def json_attributes
+          Hash[attributes.map { |name, val| [name, val.as_json] }]
+        end
+
+        private
+
+        def validate_not_base_class
+          if instance_of?(Messages::Message)
+            errors.add(:base, "base Message class is abstract")
+          end
+        end
+      end
+    end
+  end
+end
+

data/lib/codeclimate/collectors/messages_facade.rb

@@ -0,0 +1,28 @@
+module Codeclimate
+  module Collectors
+    class MessagesFacade
+      # Wraps functionality for sending messages. Takes care of some basic
+      # validation and such so that individual implementations don't need to.
+      #
+      # +implementation+ should respond to +#send_message+.
+      def initialize(implementation:)
+        @implementation = implementation
+      end
+
+      def send_message(message)
+        if !message.valid?
+          raise Messages::InvalidMessage, message.errors.full_messages.to_sentence
+        end
+        implementation.send_message(message)
+      end
+
+      def <<(message)
+        send_message(message)
+      end
+
+      private
+
+      attr_reader :implementation
+    end
+  end
+end

data/lib/codeclimate/collectors/testing.rb

@@ -0,0 +1,17 @@
+require "codeclimate/collectors/testing/stub_messages_facade"
+
+module Codeclimate
+  module Collectors
+    module Testing
+      def stub_manager
+        Codeclimate::Collectors::Manager.new(
+          messages: stub_messages_facade,
+        )
+      end
+
+      def stub_messages_facade
+        Codeclimate::Collectors::Testing::StubMessagesFacade.new
+      end
+    end
+  end
+end

data/lib/codeclimate/collectors/testing/stub_messages_facade.rb

@@ -0,0 +1,25 @@
+module Codeclimate
+  module Collectors
+    module Testing
+      class StubMessagesFacade < MessagesFacade
+        attr_reader :received_messages
+
+        class NullMessagesImplementation
+          def send_message(_msg); end
+        end
+
+        def initialize(implementation: NullMessagesImplementation.new)
+          super
+
+          @received_messages = []
+        end
+
+        def send_message(message)
+          super
+          received_messages << message
+        end
+      end
+    end
+  end
+end
+

data/lib/codeclimate/collectors/validations/type_validator.rb

@@ -0,0 +1,14 @@
+module Codeclimate
+  module Collectors
+    module Validations
+      class TypeValidator < ::ActiveModel::EachValidator
+        def validate_each(record, attr_name, value)
+          type = options.fetch(:type)
+          if !value.nil? && !value.is_a?(type)
+            record.errors.add(attr_name, "must be a #{type}")
+          end
+        end
+      end
+    end
+  end
+end

metadata

@@ -0,0 +1,99 @@
+--- !ruby/object:Gem::Specification
+name: codeclimate-collector-manager
+version: !ruby/object:Gem::Version
+  version: 0.0.2
+platform: ruby
+authors:
+- Code Climate
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2020-02-18 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: activemodel
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.6
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 5.1.6
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Interfaces for collectors to use
+email:
+- hello@codeclimate.com
+executables:
+- run-collector
+extensions: []
+extra_rdoc_files: []
+files:
+- bin/run-collector
+- lib/codeclimate-collector-manager.rb
+- lib/codeclimate/collectors.rb
+- lib/codeclimate/collectors/configuration.rb
+- lib/codeclimate/collectors/manager.rb
+- lib/codeclimate/collectors/messages.rb
+- lib/codeclimate/collectors/messages/configuration_verification.rb
+- lib/codeclimate/collectors/messages/incident.rb
+- lib/codeclimate/collectors/messages/message.rb
+- lib/codeclimate/collectors/messages_facade.rb
+- lib/codeclimate/collectors/testing.rb
+- lib/codeclimate/collectors/testing/stub_messages_facade.rb
+- lib/codeclimate/collectors/validations/type_validator.rb
+homepage: https://codeclimate.com
+licenses:
+- Nonstandard
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 3.1.2
+signing_key: 
+specification_version: 4
+summary: Code Climate Collector Manager
+test_files: []