yabeda-datadog 0.1.0 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 76499b2fc49b319aba467fc5ae335b8eb7a8e978cfb2ab1a5b659ba47d7b1dfa
-  data.tar.gz: 43c14e0c4f9db28ddb299049cf5c06aeeac5a97ccdb1e3411f71edeb98f2456e
+  metadata.gz: a61ee9350adf2ceab80125e940127a919d1fcef9ce84444a41644d689182df0e
+  data.tar.gz: 9f941017b4fa519c18a3ed0c4351c76990464a7b9098ab6031fc7f008da8537d
 SHA512:
-  metadata.gz: fd582c08f725d7e158119e54ac5b3d6d7edb3d3434725c7652c76d86dec305e09d419bb26bc26392a4e2cf4ec3d9b5987a25b69e5fc330853a22c29910923ad2
-  data.tar.gz: dcd61b99c6cf2dcec3ba6aa09678b839cb445315a83351a9744840650a4b4d6993ead2521d72736ac89812e01436a44d45eeb009f57e157dfce3f45e15ef098a
+  metadata.gz: 97ba2ef03616154f4ae5ff48bad10cda63e8625bd25bc706babd3ab63d26d0b733cdbc831ea5b0ab84c06ea073571e468faab8a7ff2933ebb9188a27ba4709f6
+  data.tar.gz: 63cf3ba1a667d8b044fd69004397d7e7579a4a31fd2fff247ef8cb781e0de091928c55660426fe611754b809a48005784e52269a2a7092149fe1ccfcf0f654b3

data/.gitignore

@@ -9,3 +9,6 @@
 
 # rspec failure tracking
 .rspec_status
+
+# environment files
+.datadog-agent.env

data/Gemfile.lock

@@ -3,16 +3,18 @@ PATH
   specs:
     yabeda-datadog (0.1.0)
       dogapi
+      dogstatsd-ruby
       yabeda
 
 GEM
   remote: https://rubygems.org/
   specs:
     coderay (1.1.2)
-    concurrent-ruby (1.1.3)
+    concurrent-ruby (1.1.4)
     diff-lcs (1.3)
     dogapi (1.32.0)
       multi_json
+    dogstatsd-ruby (4.0.0)
     dry-initializer (2.5.0)
     method_source (0.9.2)
     multi_json (1.13.1)
@@ -33,7 +35,7 @@ GEM
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.8.0)
     rspec-support (3.8.0)
-    yabeda (0.1.2)
+    yabeda (0.1.3)
       concurrent-ruby
       dry-initializer
 
@@ -48,4 +50,4 @@ DEPENDENCIES
   yabeda-datadog!
 
 BUNDLED WITH
-   1.17.1
+   1.17.2

data/README.md

@@ -2,12 +2,13 @@
 
 [Yabeda](https://github.com/yabeda-rb/yabeda) adapter for easy exporting collected custom metrics from your application to the Datadog API.
 
+## Prerequisites
 
-## Installation
+Have an instance of Datadog agent and dogstats-d running. For other installation options of Datadog agent please refer to [Datadog agent documentation](https://docs.datadoghq.com/agent/).
 
-This adapter send metrics to [Datadog API](https://docs.datadoghq.com/api/?lang=ruby) and requires to have and Datadog account with API key. You can obtain your Datadog API key in [Datadog dashboard](https://app.datadoghq.com/account/settings#api). For more information refer to [API documentation](https://docs.datadoghq.com/api/?lang=ruby#authentication).
+## Installation
 
-Add line to your application's Gemfile:
+Add to your application's Gemfile:
 
 ```ruby
 gem 'yabeda-datadog'
@@ -19,22 +20,38 @@ And then execute:
 
 ## Usage
 
-Require this adapter before Yabeda configuration:
+Configure Yabeda metrics. Refer to [Yabeda documentation](https://github.com/yabeda-rb/yabeda) for instruction on how to configure and use Yabeda metrics.
 
-```ruby
-require "yabeda/datadog"
-```
-
-Refer to [Yabeda documentation](https://github.com/yabeda-rb/yabeda) for instruction how to configure and use Yabeda.
+Please note when configuring Yabeda you have to use Datadog units. Refer to [Datadog unit for metrics documentation](https://docs.datadoghq.com/developers/metrics/#units).
+If a unit of a metric is not supported by Datadog, this unit will not be automatically updated. But you always have the ability to update it manually in Datadog metrics dashboard or by calling API by yourself.
 
 Refer to [Datadog metrics documentation](https://docs.datadoghq.com/graphing/metrics/) for working with your metrics in Datadog dashboard.
 
-Please note that configuring Yabeda you have to use Datadog units. Refer to [Datadog unit for metrics documentation](https://docs.datadoghq.com/developers/metrics/#units).
+Start your application. You have to set your `DATADOG_API_KEY` and `DATADOG_APP_KEY` as environment variables.
+You can obtain your Datadog API keys in [Datadog dashboard](https://app.datadoghq.com/account/settings#api).
+
+You may specify `DATADOG_AGENT_HOST` and/or `DATADOG_AGENT_PORT` environment variables if your Datadog agent is run not in the same host as an app/code that you collection metrics.
+
+You may specify `YABEDA_DATADOG_COLLECT_INTERVAL` environment variable to change default interval to call Yabeda collect blocks (aka collectors).
+
+### Limitations
+
+On the first run of your application no metrics metadata will be updated. This is happening because metrics have not yet been collected, thus not been created, and there is nothing to update.
+
+### Example
+
+[yabeda-datadog-sidekiq-example](https://github.com/shvetsovdm/yabeda-datadog-sidekiq-example)
 
 ## Development
 
 After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment.
 
+In can run a dogstats-d instance in a docker container with the following command:
+
+    $ bin/dev
+
+Beware that the agent will collect metrics (a lot) from docker itself and all launched docker containers.
+
 To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org).
 
 ## Contributing

data/Rakefile

@@ -1,6 +1,8 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 require "rspec/core/rake_task"
 
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task default: :spec

data/bin/dev

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+set -e
+
+docker run --rm \
+  --env-file .datadog-agent.env \
+  -v /var/run/docker.sock:/var/run/docker.sock:ro \
+  -v /proc/:/host/proc/:ro \
+  -v /sys/fs/cgroup/:/host/sys/fs/cgroup:ro \
+  -p 8125:8125/udp \
+  datadog/agent:latest

data/examples/script.rb

@@ -3,15 +3,15 @@
 require "bundler/setup"
 require "yabeda/datadog"
 
-# Put Datadog an API key in DATADOG_API_KEY env variable
-# and appliction key in DATADOG_APP_KEY env variable.
-# Refer to Datadog integrations dashboard for an API key
+Thread.abort_on_exception = true
 
 # To Use this script execute it directly with ruby command.
+# You have to provide DATADOG_API_KEY and DATADOG_APP_KEY
+# environment variables.
 #
 # Example:
 #
-#   DATADOG_API_KEY="your-datadog-api-key-here" ruby script.rb
+#   DATADOG_API_KEY=<your API key> DATADOG_APP_KEY=<your app key> ruby script.rb
 #
 
 Yabeda.configure do
@@ -21,6 +21,6 @@ Yabeda.configure do
 end
 
 start_time = Time.now
-Yabeda.yabeda_datadog_gem_examples_script_run_count.increment(device: "developent_computer")
+Yabeda.yabeda_datadog_gem_examples_script_run_count.increment(host: "dev_machine")
 finish_time = Time.now
-Yabeda.yabeda_datadog_gem_examples_script_run_time.set({ device: "developent_computer" }, finish_time - start_time)
+Yabeda.yabeda_datadog_gem_examples_script_run_time.set({ host: "dev_machine" }, finish_time - start_time)

data/lib/yabeda/datadog/adapter.rb

@@ -1,84 +1,83 @@
 # frozen_string_literal: true
 
-require "dogapi"
+require "yabeda/datadog/metric"
+require "yabeda/datadog/tags"
 require "yabeda/base_adapter"
+require "datadog/statsd"
+require "dogapi"
 
 module Yabeda
   module Datadog
-    # DataDog adapter. Sends yabeda metrics as custom metrics to DataDog API.
+    DEFAULT_AGENT_HOST = "localhost"
+    DEFAULT_AGENT_PORT = 8125
+
+    # = DataDog adapter.
+    #
+    # Sends yabeda metrics as custom metrics to DataDog API.
     # https://docs.datadoghq.com/integrations/ruby/
     class Adapter < BaseAdapter
       def register_counter!(counter)
-        metric = AdapterMetric.new(counter, "counter")
-        dog.update_metadata(metric.name, metric.metadata)
+        metric = Metric.new(counter, "counter")
+        Thread.new { metric.update(dogapi) }
       end
 
       def perform_counter_increment!(counter, tags, increment)
-        metric = AdapterMetric.new(counter, "counter")
-        tags[:type] = "counter"
-        dog.emit_point(metric.name, increment, tags)
+        metric = Metric.new(counter, "counter")
+        dogstatsd.count(metric.name, increment, tags: Tags.build(tags))
       end
 
       def register_gauge!(gauge)
-        metric = AdapterMetric.new(gauge, "gauge")
-        dog.update_metadata(metric.name, metric.metadata)
+        metric = Metric.new(gauge, "gauge")
+        Thread.new { metric.update(dogapi) }
       end
 
       def perform_gauge_set!(gauge, tags, value)
-        metric = AdapterMetric.new(gauge, "gauge")
-        tags[:type] = "gauge"
-        dog.emit_point(metric.name, value, tags)
+        metric = Metric.new(gauge, "gauge")
+        dogstatsd.gauge(metric.name, value, tags: Tags.build(tags))
       end
 
-      private
-
-      def dog
-        ::Dogapi::Client.new(ENV["DATADOG_API_KEY"], ENV["DATADOG_APP_KEY"])
-      end
-
-      Yabeda.register_adapter(:datadog, new)
-    end
-
-    # Internal adapter representation of metrics
-    class AdapterMetric
-      def initialize(metric, type)
-        @metric = metric
-        @type = type
+      def register_histogram!(histogram)
+        # sending many requests in separate threads
+        # cause rejections by Datadog API
+        Thread.new do
+          histogram_metrics(histogram).map do |historgam_sub_metric|
+            historgam_sub_metric.update(dogapi)
+          end
+        end
       end
 
-      attr_reader :type
-
-      def metadata
-        {
-          type: type,
-          description: description,
-          short_name: name,
-          unit: unit,
-          per_unit: per_unit,
-        }
+      def perform_histogram_measure!(historam, tags, value)
+        metric = Metric.new(historam, "histogram")
+        dogstatsd.histogram(metric.name, value, tags: Tags.build(tags))
       end
 
-      def name
-        parts = ""
-        parts += "#{metric.group}." if metric.group
-        parts + metric.name.to_s
-      end
+      private
 
-      def description
-        metric.comment
+      def dogstatsd
+        # consider memoization here
+        ::Datadog::Statsd.new(
+          ENV.fetch("DATADOG_AGENT_HOST", DEFAULT_AGENT_HOST),
+          ENV.fetch("DATADOG_AGENT_PORT", DEFAULT_AGENT_PORT),
+        )
       end
 
-      def unit
-        metric.unit
+      def dogapi
+        # consider memoization here
+        ::Dogapi::Client.new(ENV["DATADOG_API_KEY"], ENV["DATADOG_APP_KEY"])
       end
 
-      def per_unit
-        metric.per
+      def histogram_metrics(historgram)
+        [
+          Metric.new(historgram, "gauge", name_sufix: "avg"),
+          Metric.new(historgram, "gauge", name_sufix: "max"),
+          Metric.new(historgram, "gauge", name_sufix: "min"),
+          Metric.new(historgram, "gauge", name_sufix: "median"),
+          Metric.new(historgram, "gauge", name_sufix: "95percentile", unit: nil, per_unit: nil),
+          Metric.new(historgram, "rate", name_sufix: "count", unit: nil, per_unit: nil),
+        ]
       end
 
-      private
-
-      attr_reader :metric
+      Yabeda.register_adapter(:datadog, new)
     end
   end
 end

data/lib/yabeda/datadog/metric.rb

@@ -0,0 +1,56 @@
+# frozen_string_literal: true
+
+require "yabeda/datadog/units"
+
+module Yabeda
+  module Datadog
+    # = Internal adapter representation of metrics
+    class Metric
+      def initialize(metric, type, overides = {})
+        @metric = metric
+        @type = type
+        @overides = overides
+      end
+
+      # Datadog API argument
+      def metadata
+        {
+          type: type,
+          description: description,
+          short_name: name,
+          unit: unit,
+          per_unit: per_unit,
+        }
+      end
+
+      # Datadog API argument
+      def name
+        [metric.group, metric.name.to_s, overides[:name_sufix]].compact.join(".")
+      end
+
+      # Datadog API argument
+      def description
+        overides.fetch(:description, metric.comment)
+      end
+
+      # Datadog API argument
+      def unit
+        overides.fetch(:unit, Unit.find(metric.unit))
+      end
+
+      # Datadog API argument
+      def per_unit
+        overides.fetch(:per_unit, Unit.find(metric.per))
+      end
+
+      # Update metric metadata
+      def update(api)
+        api.update_metadata(name, metadata)
+      end
+
+      private
+
+      attr_reader :metric, :type, :overides
+    end
+  end
+end

data/lib/yabeda/datadog/tags.rb

@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Yabeda
+  module Datadog
+    # = The logic of working with Datadog tags
+    class Tags
+      def self.build(tags)
+        tags.map { |key, value| "#{key}:#{value}" }
+      end
+    end
+  end
+end

data/lib/yabeda/datadog/units.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Yabeda
+  module Datadog
+    # = The logic of working with Datadog units
+    class Unit
+      # source: https://docs.datadoghq.com/developers/metrics/#units
+      AVAILABLE = ["bit", "byte", "kibibyte", "mebibyte", "gibibyte", "tebibyte",
+                   "pebibyte", "exbibyte", "nanosecond", "microsecond", "millisecond",
+                   "second", "minute", "hour", "day", "week", "percent_nano", "percent",
+                   "apdex", "fraction", "connection", "request", "packet", "segment",
+                   "response", "message", "payload", "timeout", "datagram", "route",
+                   "session", "process", "thread", "host", "node", "fault", "service",
+                   "instance", "cpu", "file", "inode", "sector", "block", "buffer",
+                   "error", "read", "write", "occurrence", "event", "time", "unit",
+                   "operation", "item", "task", "worker", "resource", "garbage collection",
+                   "email", "sample", "stage", "monitor", "location", "check", "attempt",
+                   "device", "update", "method", "job", "container", "execution",
+                   "throttle", "invocation", "user", "success", "build", "prediction",
+                   "table", "index", "lock", "transaction", "query", "row", "key",
+                   "command", "offset", "record", "object", "cursor", "assertion", "scan",
+                   "document", "shard", "flush", "merge", "refresh", "fetch", "column",
+                   "commit", "wait", "ticket", "question", "hit", "miss", "eviction",
+                   "get", "set", "dollar", "cent", "page", "split", "hertz", "kilohertz",
+                   "megahertz", "gigahertz", "entry", "degree celsius", "degree fahrenheit",
+                   "nanocore", "microcore", "millicore", "core", "kilocore", "megacore",
+                   "gigacore", "teracore", "petacore", "exacore",].freeze
+
+      # Find valid unit
+      def self.find(unit)
+        unit if unit && AVAILABLE.include?(unit)
+      end
+    end
+  end
+end

data/lib/yabeda/datadog/version.rb

@@ -2,6 +2,6 @@
 
 module Yabeda
   module Datadog
-    VERSION = "0.1.0"
+    VERSION = "0.2.0"
   end
 end

data/lib/yabeda/datadog.rb

@@ -5,7 +5,18 @@ require "yabeda/datadog/adapter"
 require "yabeda/datadog/version"
 
 module Yabeda
-  # Namespace for DataDog adapter
+  # = Namespace for DataDog adapter
   module Datadog
+    SECOND = 1
+    DEFAULT_COLLECT_INTERVAL = 60 * SECOND
+
+    def self.start_exporter(collect_interval: DEFAULT_COLLECT_INTERVAL)
+      Thread.new do
+        loop do
+          Yabeda.collectors.each(&:call)
+          sleep(collect_interval)
+        end
+      end
+    end
   end
 end

data/yabeda-datadog.gemspec

@@ -28,6 +28,7 @@ Gem::Specification.new do |spec|
   spec.require_paths = ["lib"]
 
   spec.add_dependency "dogapi"
+  spec.add_dependency "dogstatsd-ruby"
   spec.add_dependency "yabeda"
 
   spec.add_development_dependency "bundler", "~> 1.17"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: yabeda-datadog
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.2.0
 platform: ruby
 authors:
 - Dmitry Shvetsov <@shvetsovdm>
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-12-10 00:00:00.000000000 Z
+date: 2018-12-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: dogapi
@@ -24,6 +24,20 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: dogstatsd-ruby
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: yabeda
   requirement: !ruby/object:Gem::Requirement
@@ -112,10 +126,14 @@ files:
 - README.md
 - Rakefile
 - bin/console
+- bin/dev
 - bin/setup
 - examples/script.rb
 - lib/yabeda/datadog.rb
 - lib/yabeda/datadog/adapter.rb
+- lib/yabeda/datadog/metric.rb
+- lib/yabeda/datadog/tags.rb
+- lib/yabeda/datadog/units.rb
 - lib/yabeda/datadog/version.rb
 - yabeda-datadog.gemspec
 homepage: https://github.com/shvetsovdm/yabeda-datadog