wavefront-client 3.2.0 → 3.3.0

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    YmVlOTljMDc5ZWJkOTAwYzZiMDlhOTJhYTY3ZTI2YjRjNTBjZDliMw==
+    MDNiNGYxMWY3Y2NmNWMzM2YwY2YxNDczZDIxNGU0MGEyN2IwNDQ0MQ==
   data.tar.gz: !binary |-
-    M2Y4ZDExZTE1NDFmNjUxZTI1NGYzOTExYzQ0ZmU2YjVhMTMzMGVmZA==
+    MWI3YTFiNTc2MWJlOWY3NWMyNDgwNDg1YjlkNWZiNTU0MjdhMGRkNg==
 SHA512:
   metadata.gz: !binary |-
-    NjRmYmU4NjU2NGQ0MGIwMzU4N2Q0MjQwNzQ4NWViYTJkYTZlNjk1Njk0Mzlh
-    MWU1MTYyNDc3NzhlYzNiMDhmMjkwOWNiODExNDMwOWNlYWY5NWMzYWJhNjYw
-    ZTdiMmVlNGYzOTY1MWQxMjZjYTUyZjA4ZjZiODY2MTI0Y2ZmZjE=
+    NDUwYjRjOWE3MWU4OWUzZTkyN2NiMDhiZWE4ODQ1YWU5ZDA0MWM0Mjc2OWVm
+    NzMxYjFiMTMzMWRiMWMyMDVkOGJlN2FmZjMyNThhYWY0YzdkMzIzZTczYTcw
+    ODRkMjJhOWY4M2ZiM2I1NjVhNjBmNDJjMDA4ZmRhNzlhMWEzYzA=
   data.tar.gz: !binary |-
-    ZjUzOTZlZDlhOGY3M2NkZWU3ZDUwM2E4Y2NmNWJiNmFmZGI4MjQ4NmI1Mzk3
-    NjZiMmQ1MTE5MzAwYmRhN2NkZTI0NTVmYjkyNDM0YzNiNWViMWNiYjExNTE1
-    YmQ2ZTIzNzVmMTY4ZDJmNjI2YmQ1ZDg2NzYxMzIzZjJmYTY4N2M=
+    YTUzNTMxMzg1OTIwYzdmYWUzMDNiOTZkOGM3NzI2NTA3OWFkM2E5YzBiOGRm
+    ODZiNjhiNzNhMjYzODZkMDgwZDJkMGZmNGM0NmQ0NjBmYTEzMWIzNGM1ZDFl
+    MTU4Njk1NjFkMWRiYjBhMWM1OWEwMjFiYjA3Yzk4OGRiOThiN2E=

data/.travis.yml

@@ -5,6 +5,7 @@ rvm:
 - 2.0.0
 - 2.1.0
 - 2.2.2
+- 2.3.0
 deploy:
   provider: rubygems
   api_key:

data/README-cli.md

@@ -12,7 +12,6 @@ The following options are valid in almost all contexts.
 ```
 -c, --config=FILE    path to configuration file [default: ~/.wavefront]
 -P, --profile=NAME   profile in configuration file [default: default]
--E, --endpoint=URI   cluster endpoint [default: metrics.wavefront.com]
 -t, --token=TOKEN    Wavefront authentication token
 -D, --debug          enable debug mode
 -h, --help           show help for command
@@ -32,6 +31,8 @@ Usage:
             [-X bool] <query>
 
 Options:
+  -E, --endpoint=URI            cluster endpoint [default:
+                                metrics.wavefront.com]
   -S, --seconds                 query granularity of seconds
   -m, --minutes                 query granularity of minutes
   -H, --hours                   query granularity of hours
@@ -109,6 +110,7 @@ Usage:
             [-f format] [-p tag] [ -s tag] <state>
 
 Options:
+  -E, --endpoint=URI   cluster endpoint [default: metrics.wavefront.com]
   -f, --format=STRING  output format (ruby, json, human)
                        [default: human]
   -p, --private=TAG    retrieve only alerts with named private tags,
@@ -167,6 +169,7 @@ Usage:
   wavefront event --help
 
 Options:
+  -E, --endpoint=URI   cluster endpoint [default: metrics.wavefront.com]
   -i, --instant        create an instantaneous event
   -V, --verbose        be verbose
   -s, --start=TIME     time at which event begins
@@ -238,6 +241,110 @@ Closing event 'puppet_run'. [2016-06-27 19:25:16 +0100]
 Removing state file /var/tmp/wavefront/events/rob/1467051916712::puppet_run.
 ```
 
+## `write` Mode: Sending Points to Wavefront
+
+The `write` command is used to put data points into Wavefront. It is
+different from other commands in that it communicates with a
+**proxy** rather than with the Wavefront API. This means it does
+not require any credentials.
+
+```
+Usage:
+  wavefront write point [-DV] [-c file] [-P profile] [-E proxy] [-t time]
+           [-p port] [-H host] [-n] [-T tag...] <metric> <value>
+  wavefront write file [-DV] [-c file] [-P profile] [-E proxy] [-H host]
+           [-p port] [-n] [-F format] [-m metric] [-T tag...] <file>
+
+Options:
+  -E, --proxy=URI      proxy endpoint [default: wavefront]
+  -t, --time=TIME      time of data point (omit to use current time)
+  -H, --host=STRING    source host [default: box]
+  -p, --port=INT       Wavefront proxy port [default: 2878]
+  -T, --tag=TAG        point tag in key=value form
+  -F, --format=STRING  format of input file or stdin [default: tmv]
+  -n, --noop           show the metric without sending it
+  -m, --metric=STRING  the metric path to which contents of a file will be
+                       assigned. If the file contains a metric name, the two
+                       will be concatenated
+  -V, --verbose        be verbose
+```
+
+`write` has two sub-commands: `write point` and `write file`. Both
+allow you to specify the proxy address and port either with
+command-line switches, or by the `proxy` and `port` values in your
+`.wavefront` file.
+
+### `write point`
+
+This provides a very quick method of putting a point into Wavefront.
+It takes two mandatory arguments: the metric path, and the metric
+value.
+
+You can optionally add point tags with multiple uses of `-T
+key=val`, and specify a timestamp with the `-t` option.
+
+### `write file`
+
+`write file` takes a whitespace-separated file, and turns it into a
+series of points, which it sends to a Wavefront proxy. Each line
+will be mapped to a single point.
+
+Each line in the file must be of the same format, and must contain a
+value. It can optionally contain metric path, a timestamp, and point
+tags. The format of the file is passed in with the `-F` option, and
+it must contain only the letters `t` (timestamp), `m` (metric path),
+`v` (value) and `T` (point tags). If `T` is used, it must come
+last: this allows you to have as many point tags as you like, and
+you do not have to have the same number for each point.
+
+The metric can also be, to some extent, described by options. If you
+do not have a metric path in the file, you can use `-m` to supply a
+path to which every point will be assigned. If you use `-m` *and* a
+field in the file, the two will be concatenated, with `-m` used as a
+global prefix.
+
+Similarly, a global timestamp can be supplied with `-t` (timestamps
+in files must be in epoch seconds, but `-t` can be any `strptime()`
+parseable string), and global point tags with one or more `-T
+key=val`s. If you supply tags with `-T` and in the file, the points
+will get both.
+
+The input file does not have to be on disk: following the Unix
+convention, you can use `-` as the filename, and `wavefront` will
+read from standard in, converting lines to points as it receives
+them. All the same rules apply as with standard files.
+
+`wavefront write file` takes some efforts to protect the user from
+sending bad data. It will not allow metrics with less than two
+components, and it will not permit timestamps prior to 2000-01-01,
+or more than a year in the future. It also checks that every
+potential data point conforms to the limits described in the
+Wavefront wire format documentation.
+
+### Examples
+
+Tell Wavefront that the value of `dev.myvalue` at this moment is
+123.
+
+```
+$ wavefront write point dev.myvalue 123
+```
+
+Write a file of retrospective data, whwere the fields are, in order,
+a timestamp, the metric path, and the value. Tag all points with
+`mytag` set to `my value`.
+
+```
+$ wavefront write file -F tmvT -T mytag="my value" datafile
+```
+
+The command `parabola.rb` prints a timestamp and a 'y' value every
+second. Plot the parabola in Wavefront.
+
+```
+$ parabola.rb | wavefront write file -F tv -m cli.demo.parabola -
+```
+
 ## Notes on Options
 
 ### Times

data/bin/wavefront

@@ -18,11 +18,12 @@ require 'pathname'
 require 'wavefront/client'
 require 'wavefront/cli'
 require 'docopt'
+require 'socket'
 include Wavefront::Constants
 
 def sanitize_keys(hash)
   hash.each_with_object({}) do |(k, v), aggr|
-    aggr[k.gsub(/-/, '').to_sym] = v
+    aggr[k.delete('-').to_sym] = v
   end
 end
 
@@ -31,12 +32,10 @@ DEF_CF = Pathname.new(ENV['HOME']) + '.wavefront'
 
 # The global_opts are available in every command.
 #
-global_opts = %Q(
+global_opts = %(
 Global options:
   -c, --config=FILE    path to configuration file [default: #{DEF_CF}]
   -P, --profile=NAME   profile in configuration file [default: default]
-  -E, --endpoint=URI   cluster endpoint [default: #{DEFAULT_HOST}]
-  -t, --token=TOKEN    Wavefront authentication token
   -D, --debug          enable debug mode
   -h, --help           show this message
 )
@@ -45,13 +44,15 @@ Global options:
 # commands we offer. They must include the global_opts.
 #
 usage = {
-ts: %Q(
+ts: %(
 Usage:
   #{ME} ts [-c file] [-P profile] [-E endpoint] [-t token] [-OD]
             [-S | -m | -H | -d] [-s time] [-e time] [-f format] [-p num]
             [-X bool] <query>
 #{global_opts}
 Options:
+  -E, --endpoint=URI            cluster endpoint [default: #{DEFAULT_HOST}]
+  -t, --token=TOKEN             Wavefront authentication token
   -S, --seconds                 query granularity of seconds
   -m, --minutes                 query granularity of minutes
   -H, --hours                   query granularity of hours
@@ -69,12 +70,14 @@ Options:
   -O, --includeObsoleteMetrics  include metrics unreported for > 4 weeks
 ),
 
-alerts: %Q(
+alerts: %(
 Usage:
   #{ME} alerts [-c file] [-P profile] [-E endpoint] [-t token]
             [-f format] [-p tag] [ -s tag] <state>
 #{global_opts}
 Options:
+  -E, --endpoint=URI   cluster endpoint [default: #{DEFAULT_HOST}]
+  -t, --token=TOKEN    Wavefront authentication token
   -f, --format=STRING  output format (#{ALERT_FORMATS.join(', ')})
                        [default: #{DEFAULT_ALERT_FORMAT}]
   -p, --private=TAG    retrieve only alerts with named private tags,
@@ -83,10 +86,10 @@ Options:
                        delimited.
 ),
 
-event: %Q(
+event: %(
 Usage:
   #{ME} event create [-V] [-c file] [-P profile] [-E endpoint] [-t token]
-           [-d description] [-s time] [-i | -e time] [-l level] [-t type]
+           [-d description] [-s time] [-i | -e time] [-l level] [-T type]
            [-H host] [-n] <event>
   #{ME} event close [-V] [-c file] [-P profile] [-E endpoint] [-t token]
            [<event>] [<timestamp>]
@@ -106,7 +109,33 @@ Options:
 
 View events in detail using the 'ts' command with the 'events()' function.
 ),
-default: %Q(
+
+write: %(
+Usage:
+  #{ME} write point [-DV] [-c file] [-P profile] [-E proxy] [-t time]
+           [-p port] [-H host] [-n] [-T tag...] <metric> <value>
+  #{ME} write file [-DV] [-c file] [-P profile] [-E proxy] [-H host]
+           [-p port] [-n] [-F format] [-m metric] [-T tag...] <file>
+  #{ME} write --help
+#{global_opts}
+Options:
+  -E, --proxy=URI      proxy endpoint [default: #{DEFAULT_PROXY}]
+  -t, --time=TIME      time of data point (omit to use current time)
+  -H, --host=STRING    source host [default: #{Socket.gethostname}]
+  -p, --port=INT       Wavefront proxy port [default: #{DEFAULT_PROXY_PORT}]
+  -T, --tag=TAG        point tag in key=value form
+  -F, --format=STRING  format of input file or stdin [default: #{DEFAULT_INFILE_FORMAT}]
+  -n, --noop           show the metric without sending it
+  -m, --metric=STRING  the metric path to which contents of a file will be
+                       assigned. If the file contains a metric name, the two
+                       will be concatenated
+  -V, --verbose        be verbose
+
+Files are whitespace separated, and fields can be defined with the -F
+option.  Use 't' for timestamp; 'm' for metric name; 'v' for value
+and 'T' for tags. Put 'T' last.
+),
+default: %(
 Wavefront CLI
 
 Usage:
@@ -118,6 +147,7 @@ Commands:
   ts          view timeseries data
   alerts      view alerts
   event       open and close events
+  write       send data points to a Wavefront proxy
 
 Use '#{ME} <command> --help' for further information.)
 }
@@ -126,13 +156,13 @@ Use '#{ME} <command> --help' for further information.)
 # help/option parser generation.
 #
 begin
-  opts = Docopt::docopt(usage[:default], version: '3.2.0')
+  opts = Docopt.docopt(usage[:default], version: '3.2.0')
 rescue Docopt::Exit => e
   cmd = ARGV.length > 0 ? ARGV.first.to_sym : nil
 
   if usage.keys.include?(cmd)
     begin
-      opts = sanitize_keys(Docopt::docopt(usage[cmd]))
+      opts = sanitize_keys(Docopt.docopt(usage[cmd]))
     rescue Docopt::Exit => e
       abort e.message
     end
@@ -156,6 +186,14 @@ when :event
 when :alerts
   require 'wavefront/cli/alerts'
   cli = Wavefront::Cli::Alerts.new(opts, [opts[:'<state>']])
+when :write
+  if opts[:file]
+    require 'wavefront/cli/batch_write'
+    cli = Wavefront::Cli::BatchWrite.new(opts, [opts[:'<state>']])
+  else
+    require 'wavefront/cli/write'
+    cli = Wavefront::Cli::Write.new(opts, [opts[:'<state>']])
+  end
 end
 
 begin

data/lib/wavefront/batch_writer.rb

@@ -0,0 +1,247 @@
+require 'wavefront/client/version'
+require 'wavefront/exception'
+require 'wavefront/constants'
+require 'uri'
+require 'socket'
+
+HOSTNAME = Socket.gethostname
+
+module Wavefront
+  #
+  # This class exists to facilitate sending of multiple data points
+  # to a Wavefront proxy. It sends points in native Wavefront
+  # format.
+  #
+  # When initializing the instance you can
+  # define point tags which will apply to all points sent via that
+  # instance.
+  #
+  # Though we provide methods to do it, it is the developer's
+  # responsibility to open and close the socket to the proxy. Points
+  # are sent by calling the write() method.
+  #
+  # The class keeps a count of the points the current instance has
+  # sent, dropped, and failed to send, in @summary. The socket is accessed
+  # through the instance variable @sock.
+  #
+  class BatchWriter
+    attr_reader :sock, :opts, :summary
+    include Wavefront::Constants
+
+    def initialize(options = {})
+      #
+      # options is of the form:
+      #
+      # {
+      #   tags:       a key-value hash of tags which will be applied to
+      #               every  point
+      #   proxy:      the address of the Wavefront proxy
+      #   port:       the port of the Wavefront proxy
+      #   noop:       if this is true, no proxy connection will be made,
+      #               and instead of sending the points, they will
+      #               be printed in Wavefront wire format.
+      #   novalidate: if this is true, points will not be validated.
+      #               This might make things go marginally quicker
+      #               if you have done point validation higher up in
+      #               the chain.
+      #   verbose:    if this is true, many of the methods will report
+      #               their progress.
+      #   debug:      if this is true, debugging output will be
+      #               printed.
+      # }
+      #
+      defaults = {
+        tags:       false,
+        proxy:      DEFAULT_PROXY,
+        port:       DEFAULT_PROXY_PORT,
+        noop:       false,
+        novalidate: false,
+        verbose:    false,
+        debug:      false,
+      }
+
+      @summary = { sent:     0,
+                   rejected: 0,
+                   unsent:   0,
+                 }
+      @opts = setup_options(options, defaults)
+
+      if opts[:tags]
+        valid_tags?(opts[:tags])
+        @global_tags = opts[:tags]
+      end
+    end
+
+    def setup_options(user, defaults)
+      #
+      # Fill in some defaults, if the user hasn't supplied them
+      #
+      defaults.merge(user)
+    end
+
+    def write(points = [], options = {})
+      #
+      # Points are defined as hashes of the following form:
+      # {
+      #    path:   metrics path. String. Mandatory.
+      #    value:  value of metric. Numeric. Mandatory.
+      #    ts:     timestamp as a Time or Date object.  default:
+      #            Time.now. May be omitted or false.
+      #    source: originating source of metric. default: `hostname`
+      #    tags:   optional hash of key: value point tags
+      # }
+      #
+      # Send multiple points by using an array of the above hashes.
+      #
+      unless points.is_a?(Hash) || points.is_a?(Array)
+        summary[:rejected] += 1
+        return false
+      end
+
+      points = [points] if points.is_a?(Hash)
+
+      points.each do |p|
+        p[:ts] = Time.at(p[:ts]) if p[:ts].is_a?(Integer)
+        begin
+          valid_point?(p)
+        rescue Wavefront::Exception::InvalidMetricName,
+               Wavefront::Exception::InvalidMetricValue,
+               Wavefront::Exception::InvalidTimestamp,
+               Wavefront::Exception::InvalidSource,
+               Wavefront::Exception::InvalidTag => e
+          puts 'Invalid point, skipping.' if opts[:verbose]
+          puts "Invalid point: #{p}. (#{e})" if opts[:debug]
+          summary[:rejected] += 1
+          next
+        end
+
+        send_point(hash_to_wf(p))
+      end
+      return summary[:rejected] == 0 ? true : false
+    end
+
+    def valid_point?(point)
+      #
+      # Validate a point so it conforms to the standard described in
+      # https://community.wavefront.com/docs/DOC-1031
+      #
+      return true if opts.key?(:novalidate) && opts[:novalidate]
+      valid_path?(point[:path])
+      valid_value?(point[:value])
+      valid_ts?(point[:ts]) if point[:ts]
+      valid_source?(point[:source])
+      valid_tags?(point[:tags]) if point[:tags] && point[:tags].length > 0
+      true
+    end
+
+    def valid_path?(path)
+      fail Wavefront::Exception::InvalidMetricName unless \
+        path.is_a?(String) && path.match(/^[a-z0-9\-_\.]+$/) &&
+        path.length < 1024
+      true
+    end
+
+    def valid_value?(value)
+      fail Wavefront::Exception::InvalidMetricValue unless value.is_a?(Numeric)
+      true
+    end
+
+    def valid_ts?(ts)
+      unless ts.is_a?(Time) || ts.is_a?(Date)
+        fail Wavefront::Exception::InvalidTimestamp
+      end
+      true
+    end
+
+    def valid_source?(path)
+      unless path.is_a?(String) && path.match(/^[a-z0-9\-_\.]+$/) &&
+             path.length < 1024
+        fail Wavefront::Exception::InvalidSource
+      end
+      true
+    end
+
+    def valid_tags?(tags)
+      tags.each do |k, v|
+        fail Wavefront::Exception::InvalidTag unless (k.length +
+             v.length < 254) && k.match(/^[a-z0-9\-_\.]+$/)
+      end
+      true
+    end
+
+    def hash_to_wf(p)
+      #
+      # Convert the hash received by the write() method to a string
+      # conforming with that defined in
+      # https://community.wavefront.com/docs/DOC-1031
+      #
+      fail ArgumentError unless p.key?(:path) && p.key?(:value) &&
+                                p.key?(:source)
+
+      m = [p[:path], p[:value]]
+      m.<< p[:ts].to_i.to_s if p.key?(:ts) && p[:ts]
+      m.<< 'source=' + p[:source]
+      m.<< tag_hash_to_str(p[:tags]) if p.key?(:tags) && p[:tags]
+      m.<< tag_hash_to_str(opts[:tags]) if opts[:tags]
+      m.join(' ')
+    end
+
+    def tag_hash_to_str(tags)
+      #
+      # Convert a hash of tags into a string of key="val" tags. The
+      # quoting is recommended in the WF wire-format guide. No tag
+      # validation is done here: we assume you used valid_tags()
+      #
+      return '' unless tags.is_a?(Hash)
+      tags.map { |k, v| "#{k}=\"#{v}\"" }.join(' ')
+    end
+
+    def send_point(point)
+      #
+      # Send a point, which should already be in Wavefront wire
+      # format.
+      #
+      if opts[:noop]
+        puts "Would send: #{point}"
+        return
+      end
+
+      puts "Sending: #{point}" if opts[:verbose] || opts[:debug]
+
+      begin
+        sock.puts(point)
+        summary[:sent] += 1
+        return true
+      rescue
+        summary[:unsent] += 1
+        puts 'WARNING: failed to send point.'
+        return false
+      end
+    end
+
+    def open_socket
+      #
+      # Open a socket to a Wavefront proxy, putting the descriptor
+      # in instance variable @sock.
+      #
+      if opts[:noop]
+        puts 'No-op requested. Not opening connection to proxy.'
+        return true
+      end
+
+      puts "Connecting to #{opts[:proxy]}:#{opts[:port]}." if opts[:verbose]
+
+      begin
+        @sock = TCPSocket.new(opts[:proxy], opts[:port])
+      rescue
+        raise Wavefront::Exception::InvalidEndpoint
+      end
+    end
+
+    def close_socket
+      return if opts[:noop]
+      puts 'Closing connection to proxy.' if opts[:verbose]
+      sock.close
+    end
+  end
+end