tsd_metrics 0.2.0 → 0.2.8

data/doc/query-log-schema-2e.json

@@ -0,0 +1,103 @@
+{
+    "$schema": "http://json-schema.org/draft-04/schema#",
+    "definitions": {
+        "sampleObject" : {
+            "type": "object",
+            "properties" : {
+                "unit": {
+                    "type": "string",
+                    "enum": ["nanosecond","microsecond", "millisecond", "second", "minute", "hour", "day", "week", "bit", "byte", "kilobit", "kilobyte", "megabit", "megabyte", "gigabit", "gigabyte", "terabyte", "petabyte"]
+                },
+                "value": {
+                    "type": "number"
+                }
+            },
+            "required": ["value"]
+        },
+        "dataElement": {
+            "type": "object",
+            "properties": { 
+                "values": {
+                    "type": "array",
+                    "items": {
+                        "$ref": "#/definitions/sampleObject"
+                    } 
+                }
+            },
+            "required": ["values"]
+        },
+        "metricsList": {
+            "type": "object",
+            "additionalProperties": {
+                "$ref": "#/definitions/dataElement"
+            }
+        },
+        "data": {
+            "type": "object",
+            "properties":{
+                "annotations": {
+                    "type":"object",
+                    "properties": {
+                        "finalTimestamp": {
+                            "type":"string",
+                            "format": "date-time"
+                        },
+                        "initTimestamp": {
+                            "type":"string",
+                            "format": "date-time"
+                        }
+                    },
+                    "required": ["initTimestamp", "finalTimestamp"],
+                    "additionalProperties": {
+                        "type": "string"
+                    }
+                },
+                "counters":  {
+                    "$ref": "#/definitions/metricsList"
+                },
+                "gauges": {
+                    "$ref": "#/definitions/metricsList"
+                },
+                "timers": {
+                    "$ref": "#/definitions/metricsList"
+                },
+                "version": {
+                    "type":"string",
+                    "pattern": "^2e$"
+                }
+            },
+            "required": ["annotations", "version"]
+        }
+    },
+
+    "title": "Query Log",
+    "description": "log file entry for ingestion by tsd aggregator",
+    "type":"object",
+
+    "properties":{
+        "time": {
+            "type":"string",
+            "format": "date-time"
+        },
+        "name":  {
+            "type":"string",
+            "pattern": "^aint.metrics$"
+        },
+        "level":  {
+            "type":"string",
+            "pattern": "^info$"
+        },
+        "data": {
+            "$ref": "#/definitions/data"
+        },
+        "id":  {
+            "type":"string"
+        },
+        "context": {
+            "type":"object",
+            "properties": {
+            }
+        }
+    },
+    "required": ["time", "name", "level", "data"]
+}

data/lib/tsd_metrics/async_queue_writer.rb

@@ -0,0 +1,40 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  # Not threadsafe
+  class AsyncQueueWriter
+    def initialize(queue, logger)
+      @queue = queue
+      @logger = logger
+    end
+
+    def start
+      Thread.new do
+        while true
+          tryPopQueueToFile
+        end
+      end
+    end
+
+    private
+
+    def tryPopQueueToFile
+      line = @queue.pop
+      return if line == nil
+      @logger << line
+      @logger << "\n"
+    end
+  end
+end

data/lib/tsd_metrics/counter.rb

@@ -0,0 +1,43 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+require_relative 'counter_sample'
+
+module TsdMetrics
+  class Counter
+
+    def initialize(parentMetric)
+      @parentMetric = parentMetric
+      @samples = []
+      @staticSample = nil
+    end
+
+    def createNewSample
+      newSample = CounterSample.new(@parentMetric)
+      @samples.push(newSample)
+      return newSample
+    end
+
+    def values
+      @samples.map do |s|
+        s.sampleRepresentation
+      end
+    end
+
+    private 
+
+    def getFirstOrNewStaticSample
+    end
+  end
+end

data/lib/tsd_metrics/counter_sample.rb

@@ -0,0 +1,37 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  class CounterSample
+    attr_reader :value
+    def initialize(metricStatusSupplier)
+      @metricStatusSupplier = metricStatusSupplier
+      @value = 0
+    end
+    def increment(magnitude = 1)
+      if @metricStatusSupplier.metricIsClosed
+        TsdMetrics.errorLogger.warn("Increment or decrement called on Counter after metric has been closed")
+        return
+      end
+      @value += magnitude
+    end
+    def decrement(magnitude = 1)
+      increment(-1*magnitude)
+    end
+    def sampleRepresentation
+      # Always unitless
+      {value: @value}
+    end
+  end
+end

data/lib/tsd_metrics/exceptions.rb

@@ -0,0 +1,22 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  class Error < RuntimeError
+  end
+  class MetricClosedError < Error
+  end
+  class OutputThreadError < Error
+  end
+end

data/lib/tsd_metrics/json_formatting_sink.rb

@@ -0,0 +1,63 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+require 'json'
+require 'time'
+
+module TsdMetrics
+  class JsonFormattingSink # Implements metricSink
+    def initialize(outputStream)
+      @outputStream = outputStream
+    end
+    def receive(tsdMetricEvent)
+      hash = {
+        time: Time.now.utc.iso8601(3),
+        name: "aint.metrics",
+        level: "info",
+        data: {
+          version: "2e",
+          gauges: proxyValuesProperty(tsdMetricEvent.gauges),
+          timers: proxyValuesProperty(tsdMetricEvent.timers),
+          counters: proxyValuesProperty(tsdMetricEvent.counters),
+          annotations: tsdMetricEvent.annotations
+        }
+      }
+      haveMetrics = [:gauges, :timers, :counters].any? do |metricType|
+        hash[:data][metricType].length > 0
+      end
+      # The timestamp annotations are always present, but we're looking for
+      # any further annotations.
+      haveMetrics = true if hash[:data][:annotations].length > 2
+
+      return unless haveMetrics
+
+      hash[:data][:annotations][:initTimestamp] = hash[:data][:annotations][:initTimestamp].utc.iso8601(3)
+      hash[:data][:annotations][:finalTimestamp] = hash[:data][:annotations][:finalTimestamp].utc.iso8601(3)
+      @outputStream.write(hash.to_json)
+    end
+
+    def record(tsdMetricEvent)
+      receive(tsdMetricEvent)
+    end
+
+    private
+
+    def proxyValuesProperty(hash)
+      hash.merge(hash) do |key, val, _|
+        {values: val}
+      end
+    end
+
+  end
+end

data/lib/tsd_metrics/metric_builder_for_single_struct_receiver.rb

@@ -0,0 +1,24 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  class MetricBuilderForSingleStructReceiver
+    def initialize(structReceiver)
+      @structReceiver = structReceiver
+    end
+    def build
+      TsdMetric.new(Time.now, @structReceiver, Mutex.new)
+    end
+  end
+end

data/lib/tsd_metrics/queue_writer.rb

@@ -0,0 +1,24 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  class QueueWriter
+    def initialize(queue)
+      @queue = queue
+    end
+    def write(s)
+      @queue << s
+    end
+  end
+end

data/lib/tsd_metrics/timer.rb

@@ -0,0 +1,48 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+require_relative 'timer_sample'
+
+module TsdMetrics
+  class Timer
+    def initialize(parentMetric)
+      @parentMetric = parentMetric
+      @samples = []
+    end
+
+    def createNewSample
+      sample = TimerSample.new(@parentMetric)
+      @samples.push sample
+      sample
+    end
+
+    def addDuration(duration)
+      sample = TimerSample.new(@parentMetric)
+      sample.duration = duration
+      @samples.push sample
+    end
+
+    def samples
+      durations = []
+      @samples.each do |s|
+        if s.stopped?
+          durations.push s.sampleRepresentation
+        else
+          TsdMetrics.errorLogger.warn("Unstopped timer dropped from log")
+        end
+      end
+      durations
+    end
+  end
+end

data/lib/tsd_metrics/timer_sample.rb

@@ -0,0 +1,69 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+module TsdMetrics
+  class TimerSample
+    attr_reader :duration, :unit
+
+    def initialize(metricStatusSupplier)
+      @metricStatusSupplier = metricStatusSupplier
+      @startTime = Time.now
+      @duration = nil
+      @unit = :nanosecond
+    end
+
+    def stop
+      if @startTime == nil
+        TsdMetrics.errorLogger.warn("Stop called on already-stopped Timer sample")
+        return
+      end
+      if @metricStatusSupplier.metricIsClosed
+        TsdMetrics.errorLogger.warn("Stop called on Timer after metric has been closed")
+        return
+      end
+      now = Time.now
+      diffSecs = now.tv_sec - @startTime.tv_sec
+      diffNanoSecs = now.tv_nsec - @startTime.tv_nsec
+      diff = ((10**9) * diffSecs) + diffNanoSecs
+      @duration = diff
+      @startTime = nil
+    end
+
+    # Deprecated: Instead use the more ruby-esque #running?
+    def isRunning
+      return running?
+    end
+
+    def running?
+      return @startTime != nil
+    end
+
+    def stopped?
+      return ! running?
+    end
+
+    def set(duration, unit)
+      @duration = duration
+      @unit = unit
+    end
+
+    def sampleRepresentation
+      if @unit == :noUnit
+        {value: @duration}
+      else
+        {value: @duration, unit: @unit}
+      end
+    end
+  end
+end

data/lib/tsd_metrics/tsd_metric.rb

@@ -0,0 +1,200 @@
+# Copyright 2014 Groupon.com
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+#     WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#     See the License for the specific language governing permissions and
+#     limitations under the License.
+
+require_relative 'timer_sample'
+require_relative 'timer'
+require_relative 'counter'
+require_relative 'exceptions'
+require_relative 'units'
+
+module TsdMetrics
+  class TsdMetric
+    # Implements TsdMetricEvent interface
+    attr_reader :annotations, :gauges
+
+    def initialize(startTime, metricSink, mutexStrategy)
+      @metricSink = metricSink
+      @annotations = {initTimestamp: startTime}
+      @mutexStrategy = mutexStrategy
+      @gauges = {}
+      @metrics = {timers: {}, counters: {}}
+      @metricClasses = {timers: Timer, counters: Counter}
+      @staticSamples = {timers: {}, counters: {}}
+      @closed = false
+    end
+
+    def open?
+      @mutexStrategy.synchronize do
+        not @closed
+      end
+    end
+
+    def setGauge(name, value, unit = :noUnit)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        assertValidUnit(unit)
+        @gauges[name] ||= []
+        @gauges[name].push({value: value, unit: unit}.select{|k, v| v != :noUnit})
+      end
+    end
+
+    def startTimer(name)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        # Timer sample is started on creation
+        pushNewStaticSample(:timers, name)
+      end
+    end
+
+    def stopTimer(name)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        sample = getStaticSample(:timers, name)
+        sample.stop
+      end
+    end
+
+    def setTimer(name, duration, unit = :noUnit)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        assertValidUnit(unit)
+        pushNewStaticSample(:timers, name)
+        sample = getStaticSample(:timers, name)
+        sample.stop()
+        sample.set(duration, unit)
+      end
+    end
+
+    def createTimer(name)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        ensureMetricExists(:timers, name)
+        sample = getMetric(:timers, name).createNewSample
+        sample
+      end
+    end
+
+    def resetCounter(name)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        ensureCounterExists(name)
+        @staticSamples[:counters][name] = getMetric(:counters, name).createNewSample
+      end
+    end
+
+    def incrementCounter(name, magnitude=1)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        ensureStaticCounterSampleExists(name)
+        getStaticSample(:counters, name).increment(magnitude)
+      end
+    end
+
+    def decrementCounter(name, magnitude=1)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        incrementCounter(name, -1*magnitude)
+      end
+    end
+
+    def createCounter(name)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        ensureMetricExists(:counters, name)
+        getMetric(:counters, name).createNewSample
+      end
+    end
+
+    def annotate(name, value)
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        @annotations[name] = value
+      end
+    end
+
+    def close
+      @mutexStrategy.synchronize do
+        assertNotClosed
+        @closed = true
+        @annotations[:finalTimestamp] = Time.now()
+        @metricSink.record(self)
+      end
+    end
+
+    def timers
+      samplesHash = {}
+      getMetricsOfType(:timers).each do |timerName,timer|
+        samplesHash[timerName] = timer.samples
+      end
+      samplesHash
+    end
+
+    def counters
+      countersHash = {}
+      getMetricsOfType(:counters).each do |k,v|
+        countersHash[k] = v.values
+      end
+      countersHash
+    end
+
+    # "Implements" metricStatusSupplier
+    def metricIsClosed
+      return @closed
+    end
+
+    private
+
+    def ensureCounterExists(name)
+      ensureMetricExists(:counters, name)
+    end
+
+    def ensureMetricExists(metricType, name)
+      @metrics[metricType][name] ||= @metricClasses[metricType].new(self)
+    end
+
+    def ensureStaticSampleExists(sampleType, name)
+      ensureMetricExists(sampleType, name)
+      @staticSamples[sampleType][name] ||= getMetric(sampleType, name).createNewSample
+    end
+
+    def ensureStaticCounterSampleExists(name)
+      ensureStaticSampleExists(:counters, name)
+    end
+
+    def pushNewStaticSample(metricType, name)
+      ensureMetricExists(metricType, name)
+      @staticSamples[metricType][name] = getMetric(metricType, name).createNewSample
+    end
+
+    def getMetric(metricType, name)
+      @metrics[metricType][name]
+    end
+
+    def getMetricsOfType(metricType)
+      @metrics[metricType]
+    end
+
+    def getStaticSample(sampleType, name)
+      @staticSamples[sampleType][name]
+    end
+
+    def assertNotClosed
+      raise MetricClosedError if @closed
+    end
+
+    def assertValidUnit(unit)
+      raise MetricClosedError unless UnitsUtils.isValidUnitValue?(unit)
+    end
+  end
+end