librato-metrics-taps 0.3.7 → 0.3.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 733e9fa30670ae136a4e48449f6f31f37c7d305e
-  data.tar.gz: 567d1545148a99222bc662d5a7f11822a39e27e9
+  metadata.gz: 3e29495d0f66b5b56bcdc03ff973812145379ea2
+  data.tar.gz: 67fcbff0a5824c5825f60124117f2326b143ce3e
 SHA512:
-  metadata.gz: 2d636a85986a909aa9fd6ad720240cb03a70bad5658a6971712deb39a20efb04ef9ba116466938e69b9cc2b9c5e5290a8204868d8fc94dc09d83934fd8dd4e21
-  data.tar.gz: 1027f1e2d5b2ae9638d81f6fe97262477f51399962aaf2ee618064524b181a2267e773561f78a52ec98c8f29071cee688729036b1c400aad7464a49c4a3f7285
+  metadata.gz: 25b57a166b0292f6808de05f262e1d3572af7b4cd63e9dc256eb0afeebb077d1a5bfc536e9ef3761f03d2377685e882a44e18622ee3070210ae93439dc246f47
+  data.tar.gz: 52db5bdacad21a636e04ff454179c1e3b13d3b94c5fb2a4c509668bf3ee9871e449c216ff71e51e9503969002b31672cae9bd045e0a2366c4ba177121182d1d0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    librato-metrics-taps (0.3.7)
+    librato-metrics-taps (0.3.9)
       jmx4r (= 0.1.4)
       jruby-openssl (= 0.8.5)
       librato-metrics (~> 1.0.4)

data/README.md

@@ -1,5 +1,15 @@
 # librato-metrics-taps
 
+## Deprecation Notice
+
+**This project is being officially deprecated by the maintainers.**
+
+We recommend alternatives such as [JMXTrans](https://github.com/jmxtrans/embedded-jmxtrans) and collectd's [JMX plugin](https://collectd.org/wiki/index.php/Plugin:GenericJMX) projects for submitting your JMX metrics to Librato. These have more comprehensive coverage, require fewer external dependencies, and are more actively developed than this project.
+
+We will keep this project available on GitHub for now, but there will be no ongoing support or development of the project from this point forward.
+
+## Overview
+
 Collection of helper scripts and library routines to tap into external
 metric sources and pump those metrics into Librato's Metric Service.
 
@@ -132,6 +142,26 @@ librato-metrics-tap-jmxbeans --email "$EMAIL" --token "$TOKEN" \
     --measure-time "$MEASURE_TIME"
 ```
 
+### Daemon Mode
+
+#### Single Interval
+
+In order to poll and publish to librato every N seconds, specify `--interval <poll-interval>` option.
+
+#### Multiple Intervals
+
+In order to publish multiple YAML bean definition files in different intervals,
+you must supply an interval file, specifying a map from YAML data files to the
+period each file's metrics must be collected and published to librato. For example:
+
+```
+metrics_1min.yaml: 60
+metrics_5min.yaml: 300
+metrics_1h.yaml: 3600
+```
+
+The interval file must be specified using the `--interval-file <interval-file-path>` option. A sample interval file is contained in the examples folder.
+
 ## Contributing to librato-metrics-taps
  
 * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet

data/VERSION

@@ -1 +1 @@
-0.3.7
+0.3.9

data/bin/librato-metrics-tap-jmxbeans

@@ -42,6 +42,63 @@ def publish_beans(publisher, beans, opts)
   return r
 end
 
+def load_yaml(filename)
+  begin
+    yaml_file= File.open(filename, "r")
+  rescue => err
+    puts "Failed to open yaml file #{filename}: #{err.message}"
+    exit 1
+  end
+
+  begin
+    yaml = YAML::load(yaml_file)
+  rescue => err
+    puts "Failed to parse yaml #{filename}: #{err.message}"
+    exit 1
+  end
+  yaml_file.close
+  return yaml
+end
+
+def publish_loop(opts, publisher, interval, beans)
+  # If --interval or --interval-file has been specified,
+  # broadcast every interval seconds. We wait for
+  # interval seconds each time to ensure we broadcast
+  # on the interval
+  #
+  # We use a random stagger to ensure that we are measuring and
+  # publishing our metrics at a random point within the period. This
+  # ensures that multiple entities are not measuring and reporting
+  # at the same exact points in time.
+
+  stagger = rand(interval)
+  begin
+    t = Time.now.tv_sec
+
+    # Floor the time to the current interval
+    t2 = (t / interval) * interval
+
+    # Offset by the stagger
+    t2 += stagger
+
+    # If our stagger is < interval/2, it is possible that we
+    # went back in time. In that case, simply skip another interval
+    #
+    if t2 <= t
+      t2 += interval
+    end
+
+    sleep (t2 - t)
+    t = Time.now
+
+    # We report our measure time as the nearest interval
+    tsecs = ((t.tv_sec + (interval / 2)) / interval) * interval
+
+    publish_beans(publisher, beans, opts.merge(:measure_time => tsecs))
+  end while true
+end
+
+
 opts = Trollop::options do
   version "Version: #{Taps::version}"
   banner <<EOS
@@ -67,6 +124,7 @@ EOS
   opt :metrics_url, "Metrics URL", :short => "-r", :default => 'https://metrics-api.librato.com'
 
   opt :interval, "Run as a daemon and poll every N seconds", :short => "-i", :type => :int
+  opt :interval_file, "YAML file specifying different intervals for multiple data files (daemon mode)", :type => :string
 
   opt :ignore_missing, "Ignore missing beans/attributes", :short => "-g"
 
@@ -118,89 +176,42 @@ end
 # Load full definition
 #
 if opts[:data_file_full]
-  filename = opts[:data_file_full]
-  begin
-    beanf = File.open(filename, "r")
-  rescue => err
-    puts "Failed to open bean file #{filename}: #{err.message}"
-    exit 1
-  end
-
-  begin
-    beans = YAML::load(beanf)
-  rescue => err
-    puts "Failed to parse #{filename}: #{err.message}"
-    exit 1
-  end
-  beanf.close
+  beans = load_yaml(opts[:data_file_full])
 elsif opts[:bean_name] && opts[:data_file_attributes]
-  # Load attributes from file
-  #
-  filename = opts[:data_file_attributes]
-  begin
-    beanf = File.open(filename, "r")
-  rescue => err
-    puts "Failed to open attributes file #{filename}: #{err.message}"
-    exit 1
-  end
-
-  begin
-    attrs = YAML::load(beanf)
-  rescue => err
-    puts "Failed to parse #{filename}: #{err.message}"
-    exit 1
-  end
-  beanf.close
+  attrs = load_yaml(opts[:data_file_attributes])
 
   beans = {}
   beannames = get_beans(opts[:bean_name])
   beannames.each do |name|
     beans[name] = attrs
   end
+elsif opts[:interval_file]
+  intervals = load_yaml(opts[:interval_file])
 else
-  err "Must specify --data-file-full or --data-file-attributes"
+  err "Must specify --data-file-full or --data-file-attributes or --interval-file"
 end
 
-unless opts[:interval]
+unless opts[:interval] or opts[:interval_file]
   r = publish_beans(publisher, beans, opts)
   exit(r ? 0 : 1)
 end
 
-# If --interval has been specified, broadcast every interval
-# seconds. We wait for interval seconds each time to ensure we
-# broadcast on the interval
-#
-# We use a random stagger to ensure that we are measuring and
-# publishing our metrics at a random point within the period. This
-# ensures that multiple entities are not measuring and reporting
-# at the same exact points in time.
-
-interval = opts[:interval]
-stagger = rand(interval)
-begin
-  t = Time.now.tv_sec
-
-  # Floor the time to the current interval
-  t2 = (t / interval) * interval
-
-  # Offset by the stagger
-  t2 += stagger
-
-  # If our stagger is < interval/2, it is possible that we
-  # went back in time. In that case, simply skip another interval
-  #
-  if t2 <= t
-    t2 += interval
+if opts[:interval]
+  #single interval, single datafile
+  interval = opts[:interval]
+  publish_loop(opts, publisher, interval, beans)
+elsif opts[:interval_file]
+  #multiple intervals, multiple datafiles
+  base_dir = File.dirname(opts[:interval_file])
+  workers = []
+  intervals.each do |data_file, interval|
+    data_file_path = "#{base_dir}/#{data_file}"
+    p "Starting publisher for data file #{data_file_path} with #{interval}s interval."
+    jmx_beans = load_yaml(data_file_path)
+    workers << Thread.new{publish_loop(opts, publisher, interval.to_i, jmx_beans)}
   end
-
-  sleep (t2 - t)
-  t = Time.now
-
-  # We report our measure time as the nearest interval
-  tsecs = ((t.tv_sec + (interval / 2)) / interval) * interval
-
-  publish_beans(publisher, beans, opts.merge(:measure_time => tsecs))
-end while true
+  workers.map(&:join)
+end
 
 exit 1
 

data/examples/cassandra/cfstats-2_0_9.yaml

@@ -0,0 +1,31 @@
+# Set of attributes to grab for each CF.
+# TODO: Add comment description for each
+#
+---
+BloomFilterDiskSpaceUsed: counter
+BloomFilterFalsePositives: counter
+BloomFilterFalseRatio:
+CompressionRatio:
+DroppableTombstoneRatio:
+LiveCellsPerSlice:
+LiveDiskSpaceUsed:
+LiveSSTableCount:
+MaxRowSize:
+MeanRowSize:
+MinRowSize:
+MaximumCompactionThreshold:
+MinimumCompactionThreshold:
+MemtableColumnsCount: counter
+MemtableDataSize:
+MemtableSwitchCount: counter
+PendingTasks:
+ReadCount: counter
+RecentBloomFilterFalsePositives:
+RecentBloomFilterFalseRatio:
+RecentReadLatencyMicros:
+RecentWriteLatencyMicros:
+TotalDiskSpaceUsed: counter
+TotalReadLatencyMicros: counter
+TotalWriteLatencyMicros: counter
+UnleveledSSTables:
+WriteCount: counter

data/examples/cassandra/tpstats-1_2_5.yaml

@@ -41,7 +41,7 @@ org.apache.cassandra.db:type=StorageProxy:
 org.apache.cassandra.db:type=StorageService: 
   Load:
   CompactionThroughputMbPerSec:
-  ExceptionCount: counter
+  ExceptionCount:
   StreamThroughputMbPerSec:
 org.apache.cassandra.internal:type=AntiEntropyStage: 
   ActiveCount:

data/examples/cassandra/tpstats-2_0_9.yaml

@@ -0,0 +1,187 @@
+# Set of non-CF specific Cassandra JMX knobs
+#
+--- 
+org.apache.cassandra.db:type=BatchlogManager:
+  TotalBatchesReplayed: counter
+org.apache.cassandra.db:type=Caches:
+  KeyCacheCapacityInBytes:
+  KeyCacheEntries:
+  KeyCacheHits: counter
+  KeyCacheRecentHitRate:
+  KeyCacheRequests: counter
+  KeyCacheSize:
+  RowCacheCapacityInBytes:
+  RowCacheEntries:
+  RowCacheHits: counter
+  RowCacheRecentHitRate:
+  RowCacheRequests:
+  RowCacheSize:
+org.apache.cassandra.db:type=Commitlog: 
+  CompletedTasks: counter
+  PendingTasks:
+  TotalCommitlogSize:
+org.apache.cassandra.db:type=CompactionManager: 
+  CompletedTasks: counter
+  PendingTasks:
+  TotalBytesCompacted: counter
+  TotalCompactionsCompleted: counter
+org.apache.cassandra.db:type=StorageProxy:
+  HintsInProgress:
+  RecentRangeLatencyMicros:
+  RecentReadLatencyMicros:
+  RecentWriteLatencyMicros:
+  TotalRangeLatencyMicros: counter
+  TotalReadLatencyMicros: counter
+  TotalWriteLatencyMicros: counter
+  TotalHints: counter
+  RangeOperations: counter
+  ReadOperations: counter
+  WriteOperations: counter
+  ReadRepairAttempted:
+  ReadRepairRepairedBackground:
+  ReadRepairRepairedBlocking:
+org.apache.cassandra.db:type=StorageService:
+  Load:
+  CompactionThroughputMbPerSec:
+  ExceptionCount:
+  StreamThroughputMbPerSec:
+org.apache.cassandra.internal:type=AntiEntropyStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=CacheCleanupExecutor:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=CompactionExecutor:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=FlushWriter:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=GossipStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=HintedHandoff:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=InternalResponseStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=MemoryMeter:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=MemtablePostFlusher:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=MigrationStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=MiscStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=PendingRangeCalculator:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=ValidationExecutor:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.internal:type=StreamStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.request:type=MutationStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.request:type=ReadRepairStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.request:type=ReadStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.request:type=ReplicateOnWriteStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.request:type=RequestResponseStage:
+  ActiveCount:
+  CompletedTasks: counter
+  PendingTasks:
+  CurrentlyBlockedTasks:
+  TotalBlockedTasks: counter
+org.apache.cassandra.net:type=MessagingService:
+  RecentTotalTimouts:
+  TotalTimeouts: counter
+org.apache.cassandra.metrics:type=ClientRequestMetrics,name=ReadTimeouts:
+  Count: counter
+org.apache.cassandra.metrics:type=ClientRequestMetrics,name=ReadUnavailables:
+  Count: counter
+org.apache.cassandra.metrics:type=ClientRequestMetrics,name=WriteTimeouts:
+  Count: counter
+org.apache.cassandra.metrics:type=ClientRequestMetrics,name=WriteUnavailables:
+  Count: counter
+org.apache.cassandra.metrics:type=Client,name=connectedNativeClients:
+  Value:
+org.apache.cassandra.metrics:type=Client,name=connectedThriftClients:
+  Value:
+org.apache.cassandra.metrics:type=ClientRequest,scope=Read,name=Latency:
+  Mean:
+  95thPercentile:
+  99thPercentile:
+org.apache.cassandra.metrics:type=ClientRequest,scope=Write,name=Latency:
+  Mean:
+  95thPercentile:
+  99thPercentile:
+org.apache.cassandra.net:type=MessagingService:
+  RecentTotalTimouts:
+  TotalTimeouts: counter

data/examples/interval-file.yaml

@@ -0,0 +1,3 @@
+cassandra/cfstats-2_0_9.yaml: 300 #5 minutes resolution
+cassandra/tpstats-2_0_9.yaml: 120 #2 minutes resolution
+java-lang.yaml: 60 #1 minute resolution

data/librato-metrics-taps.gemspec

@@ -5,7 +5,7 @@ Gem::Specification.new do |s|
   s.version = File.read(File.join(File.dirname(__FILE__), 'VERSION')).chomp
 
   s.authors = [%q{Librato, Inc.}]
-  s.date = %q{2012-05-15}
+  s.date = Time.now
   s.description = %q{Taps for extracting metrics data and publishing to Librato Metrics}
   s.email = %q{mike@librato.com}
   s.summary = %q{Librato Metrics Taps}

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: librato-metrics-taps
 version: !ruby/object:Gem::Version
-  version: 0.3.7
+  version: 0.3.9
 platform: ruby
 authors:
 - Librato, Inc.
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2012-05-15 00:00:00.000000000 Z
+date: 2015-01-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: jmx4r
@@ -129,11 +129,14 @@ files:
 - examples/cassandra/cfstats-0_8_8.yaml
 - examples/cassandra/cfstats-1_1_6.yaml
 - examples/cassandra/cfstats-1_2_5.yaml
+- examples/cassandra/cfstats-2_0_9.yaml
 - examples/cassandra/tpstats-0_7_6-2.yaml
 - examples/cassandra/tpstats-0_8_1.yaml
 - examples/cassandra/tpstats-0_8_8.yaml
 - examples/cassandra/tpstats-1_1_6.yaml
 - examples/cassandra/tpstats-1_2_5.yaml
+- examples/cassandra/tpstats-2_0_9.yaml
+- examples/interval-file.yaml
 - examples/java-lang.yaml
 - lib/librato-metrics-taps.rb
 - lib/librato/metrics/taps.rb