signalfx 1.0.2 → 2.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: b1e95b59377f963bd7d58da66522048b421077cd
-  data.tar.gz: 300ab136c02605ccdf4c984be4d1d8978711868e
+  metadata.gz: 65d54216c0f3f8e9bac2bed755772ac96dc3b415
+  data.tar.gz: e90eeddd906dfe039ef213c5d605e4c29afca0da
 SHA512:
-  metadata.gz: 146a8cdb0ff97ef3027f4a545051af579860a8a46cee893186f61d6e539774537fa3031317deb7f0127747ce4ce3f915bccd5149435636278d0f07d6113cccb3
-  data.tar.gz: d45317cab75adabb588f0b67c5048007e083ac8bc557fc5827b153c55f33dd535a40e37409c9ece1b553d4078ec30b9d8ed7d5d26e0a6adbb7d3a6e0adbdb685
+  metadata.gz: 469077ec78a1414d3a44c6f8482a90d8e78ba2cf27a4a16d1708c87999baf7bf24bd761d819ca2a4f6a9d05b70c68e9f73302f509da9c50b130f16bb321f16f1
+  data.tar.gz: 183aa859b752dab6739d5a4d7e1668e1fa39b0c486197ab57c233ce159640b1f3d329653d9d3cd7d4a1be037e62e1d5f68b88ec8611962aff1efa18eb23aee97

data/.travis.yml

@@ -1,5 +1,6 @@
 sudo: required
 language: ruby
+dist: trusty
 
 rvm:
   - 2.2.3

data/Gemfile.lock

@@ -1,44 +1,58 @@
 PATH
   remote: .
   specs:
-    signalfx (1.0.1)
+    signalfx (2.0.1)
       protobuf (~> 3.5.1, >= 3.5.1)
       rest-client (~> 2.0)
+      websocket-client-simple (~> 0.3.0)
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (5.0.0.1)
+    activesupport (5.1.4)
       concurrent-ruby (~> 1.0, >= 1.0.2)
       i18n (~> 0.7)
       minitest (~> 5.1)
       tzinfo (~> 1.1)
     addressable (2.4.0)
-    concurrent-ruby (1.0.2)
+    coderay (1.1.1)
+    concurrent-ruby (1.0.5)
     crack (0.4.3)
       safe_yaml (~> 1.0.0)
+    daemons (1.2.4)
     diff-lcs (1.2.5)
     docile (1.1.5)
-    domain_name (0.5.20160826)
+    domain_name (0.5.20170404)
       unf (>= 0.0.5, < 1.0.0)
+    event_emitter (0.2.6)
+    eventmachine (1.2.5)
+    faye-websocket (0.10.7)
+      eventmachine (>= 0.12.0)
+      websocket-driver (>= 0.5.1)
     hashdiff (0.3.0)
-    http-cookie (1.0.2)
+    http-cookie (1.0.3)
       domain_name (~> 0.5)
-    i18n (0.7.0)
+    i18n (0.8.6)
     json (1.8.3)
+    method_source (0.8.2)
     middleware (0.1.0)
     mime-types (3.1)
       mime-types-data (~> 3.2015)
     mime-types-data (3.2016.0521)
-    minitest (5.9.0)
+    minitest (5.10.3)
     netrc (0.11.0)
     protobuf (3.5.5)
       activesupport (>= 3.2)
       middleware
       thor
       thread_safe
+    pry (0.10.4)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+    rack (2.0.3)
     rake (10.4.2)
-    rest-client (2.0.0)
+    rest-client (2.0.2)
       http-cookie (>= 1.0.2, < 2.0)
       mime-types (>= 1.16, < 4.0)
       netrc (~> 0.8)
@@ -61,28 +75,43 @@ GEM
       json (~> 1.8)
       simplecov-html (~> 0.10.0)
     simplecov-html (0.10.0)
-    thor (0.19.1)
-    thread_safe (0.3.5)
-    tzinfo (1.2.2)
+    slop (3.6.0)
+    thin (1.7.2)
+      daemons (~> 1.0, >= 1.0.9)
+      eventmachine (~> 1.0, >= 1.0.4)
+      rack (>= 1, < 3)
+    thor (0.20.0)
+    thread_safe (0.3.6)
+    tzinfo (1.2.3)
       thread_safe (~> 0.1)
     unf (0.1.4)
       unf_ext
-    unf_ext (0.0.7.2)
+    unf_ext (0.0.7.4)
     webmock (2.1.0)
       addressable (>= 2.3.6)
       crack (>= 0.3.2)
       hashdiff
+    websocket (1.2.4)
+    websocket-client-simple (0.3.0)
+      event_emitter
+      websocket
+    websocket-driver (0.6.5)
+      websocket-extensions (>= 0.1.0)
+    websocket-extensions (0.1.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   bundler (~> 1.10)
+  faye-websocket (~> 0.10.7)
+  pry
   rake (~> 10.0)
   rspec (~> 3.3)
   signalfx!
   simplecov
+  thin (~> 1.7)
   webmock (~> 2.1)
 
 BUNDLED WITH
-   1.13.1
+   1.15.4

data/README.md

@@ -155,6 +155,26 @@ client.send_event(
 See `examples/generic_usecase.rb` for a complete code example for
 sending events.
 
+### SignalFlow
+
+You can run SignalFlow computations as well.  This library supports all of the
+functionality described in our [API docs for
+SignalFlow](https://developers.signalfx.com/reference#signalflowconnect). Right
+now, the only supported transport mechanism is WebSockets.
+
+To create a new SignalFlow client instance from an existing SignalFx client:
+
+```ruby
+signalflow = client.signalflow()
+```
+
+For the full API see [the RubyDocs for
+the SignalFlow
+client](http://www.rubydoc.info/github/signalfx/signalfx-ruby/master/SignalFlowClient/)
+(the `signalflow` var above).
+
+There is also [a demo script](./examples/signalflow.rb) that shows basic usage.
+
 ## License
 
 Apache Software License v2. Copyright © 2015-2016

data/examples/signalflow.rb

@@ -0,0 +1,32 @@
+$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__)
+require './lib/signalfx'
+
+token = ARGV[0] # Your SignalFx API access token
+if token.nil? || token.empty?
+  puts '
+        SignalFx API access token not defined. Please specify token in command line.
+            $ ./signalflow.rb YOUR_TOKEN
+
+        '
+  exit 0
+end
+
+
+#create client instance with SignalFx API access token
+client = SignalFx.new(token, enable_aws_unique_id: false, timeout: 3000)
+
+puts 'SignalFlow demo:'
+puts
+
+signalflow = client.signalflow()
+
+signalflow.execute("data('cpu.utilization').publish()").each_message do |msg, comp|
+  case msg[:type]
+  when "data"
+    puts "#{'Host'.center(40, ' ')} | cpu.utilization"
+    msg[:data].each do |tsid,value|
+      puts "#{comp.metadata[tsid][:host][0..40].center(40, ' ')} | #{value}"
+    end
+  end
+  puts ""
+end

data/lib/signalfx/conf.rb

@@ -3,6 +3,8 @@
 module RbConfig
   # Default Parameters
   DEFAULT_INGEST_ENDPOINT = 'https://ingest.signalfx.com'
+  DEFAULT_API_ENDPOINT = 'https://api.signalfx.com'
+  DEFAULT_STREAM_ENDPOINT = 'wss://stream.signalfx.com'
   DEFAULT_BATCH_SIZE = 300 # Will wait for this many requests before posting
   DEFAULT_TIMEOUT = 1
 

data/lib/signalfx/signal_fx_client.rb

@@ -2,6 +2,7 @@
 
 require_relative './version'
 require_relative './conf'
+require_relative './signalflow/client'
 
 require 'net/http'
 require 'uri'
@@ -30,12 +31,16 @@ class SignalFxClient
   def initialize(api_token,
                  enable_aws_unique_id: false,
                  ingest_endpoint: RbConfig::DEFAULT_INGEST_ENDPOINT,
+                 api_endpoint: RbConfig::DEFAULT_API_ENDPOINT,
+                 stream_endpoint: RbConfig::DEFAULT_STREAM_ENDPOINT,
                  timeout: RbConfig::DEFAULT_TIMEOUT,
                  batch_size: RbConfig::DEFAULT_BATCH_SIZE,
                  user_agents: [])
 
     @api_token = api_token
     @ingest_endpoint = ingest_endpoint
+    @api_endpoint = api_endpoint
+    @stream_endpoint = stream_endpoint
     @timeout = timeout
     @batch_size = batch_size
     @user_agents = user_agents
@@ -161,6 +166,15 @@ class SignalFxClient
     post(build_event(data), @ingest_endpoint, EVENT_ENDPOINT_SUFFIX)
   end
 
+  # Create a new SignalFlow client.  A single client can execute multiple
+  # computations that will be multiplexed over the same WebSocket connection.
+  #
+  # @return [SignalFlowClient] a newly instantiated client, configured with the
+  #   api token and endpoints from this class
+  def signalflow
+    SignalFlowClient.new(@api_token, @api_endpoint, @stream_endpoint)
+  end
+
   protected
 
   def get_queue

data/lib/signalfx/signalflow/binary.rb

@@ -0,0 +1,62 @@
+# Copyright (C) 2017 SignalFx, Inc. All rights reserved.
+
+require 'json'
+require 'zlib'
+
+# Converts binary WebSocket messages into a hash
+module BinaryMessageParser
+  # data should be a raw string
+  def parse(data)
+    # See https://developers.signalfx.com/v2/reference#section-binary-encoding-of-websocket-messages
+    version, message_type, flags, _, channel, payload = data.unpack("CCb8CZ16a*")
+    compressed = flags[0] == "1"
+    is_json = flags[1] == "1"
+
+    if version != 1
+      raise "Unsupported SignalFlow version #{version}"
+    end
+
+    if compressed
+      payload = Zlib::Inflate.new(16+Zlib::MAX_WBITS).inflate(payload)
+    end
+
+    raise "Unknown binary message type #{message_type}" if !is_json && message_type != 5
+
+    message = is_json ?
+                JSON.parse(payload, {:symbolize_names => true}) :
+                parse_data_payload(payload)
+
+    message.merge({:channel => channel})
+  end
+  module_function :parse
+
+  def parse_data_payload(payload)
+    # See https://developers.signalfx.com/v2/reference#section-binary-encoding-used-for-the-websocket
+    timestamp, element_count, tuples_raw = payload.unpack("Q>L>a*")
+    data_hash = (0..element_count-1).map do |i|
+      type, tsid, value_raw = tuples_raw[i*17..i*17+16].unpack("CQ>a8")
+
+      value = case type
+              when 1  # long
+                value_raw.unpack("q>")
+              when 2  # double
+                value_raw.unpack("G")
+              when 3  # int (32 bit)
+                value_raw.unpack("l>")
+              end
+
+      [
+        Base64.urlsafe_encode64([tsid].pack("Q>")).gsub("=", ""),
+        value[0],
+      ]
+    end.to_h
+
+    {
+      :type => "data",
+      :logicalTimestampMs => timestamp,
+      :logicalTimestamp => Time.at(timestamp / 1000.0),
+      :data => data_hash,
+    }
+  end
+  module_function :parse_data_payload
+end

data/lib/signalfx/signalflow/channel.rb

@@ -0,0 +1,80 @@
+# Copyright (C) 2017 SignalFx, Inc. All rights reserved.
+
+require_relative './queue'
+
+class ChannelTimeout < Exception
+end
+
+# Channel represents a medium through which SignalFlow messages pass.
+# The main method for it is {#each_message}, which is how you get messages from
+# the channel.  There can only be one user of a channel and they are NOT
+# thread-safe.
+#
+# Channels are for one-time use only.  Once a channel is detached from (either
+# manually or due to the end of a computation) previous messages will be
+# iterable but nothing new will show up.
+class Channel
+  attr_accessor :name
+  attr_accessor :detached
+
+  def initialize(name, detach_cb)
+    @lock = Mutex.new
+    @detach_lock = Mutex.new
+    @detached = false
+    @name = name
+    @detach_from_transport = detach_cb
+    @messages = QueueWithTimeout.new
+  end
+
+  # Waits for and returns the next message in the channel.
+  #
+  # @param timeout_seconds [Float] Number of seconds to wait for a message.
+  #
+  # @return [Hash] The next message received by this channel.  A return value
+  # of `nil` indicates that the channel has detected it is done and will not be
+  # receiving any more useful messages.
+  #
+  # @raise [ChannelTimeout] If the timeout is exceeded with no messages
+  def pop(timeout_seconds=nil)
+    raise "Channel #{@name} is detached" if @detached
+
+    msg = nil
+    begin
+      msg = @messages.pop_with_timeout(timeout_seconds)
+    rescue ThreadError
+      raise ChannelTimeout.new(
+        "Did not receive a message on channel #{@name} within #{timeout_seconds} seconds")
+    end
+
+    if msg[:event] == "END_OF_CHANNEL" || msg[:event] == "CONNECTION_CLOSED" || msg[:event] == "CHANNEL_ABORT"
+      # Mark this channel as detached and then return nil as an indicator that
+      # this channel is done
+      detach(false)
+
+      nil
+    else
+      msg
+    end
+
+  end
+
+
+  def detach(send_detach_to_server=true)
+    if !@detached
+      @detached = true
+      @detach_from_transport.call if send_detach_to_server
+      @detach_from_transport = nil
+    end
+  end
+
+  def inject_message(msg)
+    # Since messages are injected by a separate websocket thread, they could
+    # come in after the user has detached manually from the channel.  Just
+    # silently ignore them in that case.
+    return if @detached
+    raise 'Cannot inject nil message' if msg.nil?
+
+    @messages << msg
+  end
+
+end

data/lib/signalfx/signalflow/client.rb

@@ -0,0 +1,73 @@
+# Copyright (C) 2017 SignalFx, Inc. All rights reserved.
+
+require 'thread'
+
+require_relative "./websocket"
+
+# A SignalFlow client that uses the WebSockets interface.
+#
+# See https://developers.signalfx.com/v2/reference#signalflowconnect for
+# low-level API details and information on the SignalFlow language.
+#
+# See
+# {https://github.com/signalfx/signalfx-ruby/blob/master/examples/signalflow.rb}
+# for an example script that uses the client.
+#
+# The messages passed into the `computation.each_message*` blocks will be
+# decoded forms of what is described in
+# {https://developers.signalfx.com/v2/reference#information-messages-specification
+# our API reference for SignalFlow}.  Hash keys will be symbols instead of
+# strings.
+class SignalFlowClient
+  def initialize(api_token, api_endpoint, stream_endpoint)
+    @transport = SignalFlowWebsocketTransport.new(api_token, stream_endpoint)
+  end
+
+  # Start a computation and attach to its output. If using WebSockets (the
+  # default), the channel name is handled internally so you do not need to
+  # supply it.
+  #
+  # See https://developers.signalfx.com/reference#section-execute-a-computation
+  #
+  # @option options [Fixnum] :start
+  # @option options [Fixnum] :stop
+  # @option options [Fixnum] :resolution
+  # @option options [Fixnum] :max_delay
+  # @option options [Boolean] :persistent
+  #
+  # @return [Computation] A {Computation} instance with an active channel
+  def execute(program, **options)
+    @transport.execute(program, **options)
+  end
+  
+  # Start and attach to a computation that tells how many times a detector
+  # would have fired in a time range between `start` and `stop`.
+  #
+  # See https://developers.signalfx.com/v2/reference#signalflowpreflight
+  #
+  # @param start [Fixnum]
+  # @param stop [Fixnum]
+  # @option options [Fixnum] :max_delay
+  #
+  # @return [Computation] A {Computation} instance with an active channel
+  def preflight(program, start, stop, **options)
+    @transport.preflight(program, start, stop, **options)
+  end
+
+  # Start a computation without attaching to it
+  #
+  # The `publish()` call in the program must specify a `metric` to publish the
+  # output to since you cannot currently attach to the output.
+  #
+  # Optional parameters are the same as {#execute}.
+  # @return [Computation] A {Computation} instance with a handle but without a
+  #   channel
+  def start(program, **options)
+    @transport.start(program, **options)
+  end
+
+  # Stop everything and close any open connections.
+  def close
+    @transport.close
+  end
+end

data/lib/signalfx/signalflow/computation.rb

@@ -0,0 +1,243 @@
+# Copyright (C) 2017 SignalFx, Inc. All rights reserved.
+
+
+STARTED_STATE = :started
+ABORTED_STATE = :aborted
+COMPLETED_STATE = :completed
+DATA_STREAMING_STATE = :data_streaming
+
+# Represents a SignalFlow computation/job.  A computation can have a channel
+# associated with it, but not necessarily (it could have been detached while
+# the computation is still running or never attached in the first place).  New
+# instances should only be created by the client and a Computation MUST have a
+# handle.
+class Computation
+  attr_accessor :handle
+  attr_accessor :channel
+  attr_accessor :state
+  attr_accessor :metadata
+  attr_accessor :resolution
+  attr_accessor :input_timeseries_count
+  attr_accessor :last_timestamp_seen
+
+  def initialize(handle, attach_func, stop_func)
+    @handle = handle
+    @channel = nil
+    @attach_func = attach_func
+    @stop_func = stop_func
+    @metadata = {}
+    # We can't have a handle until the job is started so we must be at least
+    # at this state
+    @state = STARTED_STATE
+
+    @pending_messages = Queue.new
+
+    @batch_size_known = false
+    @expected_batch_size = 0
+    @current_batch_data = nil
+    @current_batch_size = nil
+
+    @last_timestamp_seen = nil
+    @resolution = nil
+    @input_timeseries_count = nil
+  end
+
+  def channel=(channel)
+    @channel = channel
+  end
+
+  def attached?
+    !@channel.nil? && !@channel.detached
+  end
+
+  # Get the next message in this computation.
+  #
+  # @param timeout_seconds [Float] If a new message does not come within this
+  #   interval, raises a {ChannelTimeout} exception.  Note that this does not
+  #   mean that this function will return within this interval since there may
+  #   be messages received that are part of a larger batch.  If nil, will block
+  #   indefinitely.
+  def next_message(timeout_seconds=nil)
+    raise "Computation #{@handle} is not attached to a channel" unless @channel
+
+    msg = nil
+    while msg.nil? && !@channel.nil?
+      # process_message might return no messages if it is building up a batch
+      msg = process_message(@channel.pop(timeout_seconds))
+    end
+    return msg
+  end
+
+  # Iterates over the messages asynchronously for this computation.  A convenience
+  # function if you want to fire off multiple computations simultaneously,
+  # though not terribly efficient since it starts a new thread that spends a
+  # lot of time waiting.  However, since we don't have a way of "select"ing on
+  # computations, this is probably good enough for basic use.
+  #
+  # See {#each_message}.
+  def each_message_async(&block)
+    raise "Computation #{@handle} is not attached to a channel" unless @channel
+
+    Thread.new{ each_message(&block) }
+    return
+  end
+
+  # Call the given block with each message in the channel as they arrive.  This
+  # method will not return until the channel is detached from (either manually
+  # or due to the computation ending).
+  #
+  # Messages are queued in the channel so that none will be lost if this method
+  # is not called immediately.
+  #
+  # @yield [msg, comp] Called when a message arrives that is relevant to the
+  #   channel's computation.  The `comp` param will be set to this computation
+  #   instance for easy referencing of computation metadata and state.  `comp`
+  #   may be omitted if this reference to the computation is not needed.
+  def each_message(&block)
+    raise "Computation #{@handle} is not attached to a channel" unless @channel
+
+    while @channel
+      msg = next_message
+      block.call(msg, self)
+    end
+
+    return
+  end
+
+  # Process the given message
+  def process_message(msg)
+    # nil is like EOF for channels
+    if msg.nil?
+      @channel = nil
+      reset_current_batch
+    else
+      # Sniff messages and update computation
+      case msg[:type]
+      when "metadata"
+        @metadata[msg[:tsId]] = msg[:properties]
+        msg
+
+      when "expired-tsid"
+        @metadata.delete(msg[:tsId])
+        msg
+
+      when "control-message"
+        case msg[:event]
+        when "CHANNEL_ABORT"
+          @state = ABORTED_STATE
+        when "END_OF_CHANNEL"
+          @state = COMPLETED_STATE
+        end
+
+        msg
+
+      when "message"
+        # Don't let users see any messages of this type, but use them to update
+        # computation state that the user can access.
+        case msg[:messageCode]
+        when 'JOB_RUNNING_RESOLUTION'
+          @resolution = msg[:contents][:resolutionMs]
+        when 'FETCH_NUM_TIMESERIES'
+          @input_timeseries_count += msg[:numInputTimeSeries]
+        end
+
+        # The server guarantees that an initial batch of data will be sent before
+        # the first "message" message.  Therefore, when we see a message of this
+        # type, we know we have determined the batch size.
+        @batch_size_known = true
+        # We also know that the current batch (if any) is done
+        reset_current_batch
+
+      when "data"
+        @state = DATA_STREAMING_STATE
+
+        # The expected batch size is the number of data messages received before
+        # either the first arrival of a "message" message or receiving two data
+        # messages with different logical timestamps.
+        if !@batch_size_known
+          @expected_batch_size += 1
+        end
+
+        out = nil
+        if @current_batch_data && msg.fetch(:logicalTimestampMs) != @current_batch_data.fetch(:logicalTimestampMs)
+          # Two data messages back to back with different timestamps before
+          # receiving the first "message" message indicate that the previous
+          # batch is done and our batch size is now whatever the total data
+          # messages seen up until this point.
+          @batch_size_known = true
+          out = reset_current_batch
+          add_to_current_batch(msg)
+        else
+          add_to_current_batch(msg)
+          if @batch_size_known && @current_batch_size == @expected_batch_size
+            out = reset_current_batch
+          end
+        end
+
+        out
+
+      when "error"
+        raise ComputationFailure.new(msg[:errors])
+
+      else
+        msg
+      end
+    end
+  end
+  private :process_message
+
+  # Add to the current batch, initializing the current batch if not already
+  # set.
+  def add_to_current_batch(msg)
+    if !@current_data_batch
+      @current_data_batch = msg
+      @current_batch_size = 1
+    else
+      @current_data_batch[:data].merge!(msg.fetch(:data))
+      @current_batch_size += 1
+    end
+  end
+  private :add_to_current_batch
+
+  # Resets the current batch, returning the previous value
+  def reset_current_batch
+    msg = @current_data_batch
+    @current_data_batch = nil
+    @current_batch_size = 0
+    @last_timestamp_seen = msg.fetch(:logicalTimestampMs) if !msg.nil?
+    msg
+  end
+  private :reset_current_batch
+
+  # Attach to an already running computation.
+  #
+  # *Not currently implemented on backend!*
+  #
+  # @return [Computation] This same computation instance with a now active
+  # channel attached to it.
+  def attach(**options)
+    raise "Computation #{@handle} is already attached!" if @channel
+
+    @channel = @attach_func.call(@handle, **options)
+    self
+  end
+
+  # Detach from this computation and remove reference to the channel to free up
+  # memory.
+  def detach
+    @channel.detach
+    @channel = nil
+  end
+
+  # Stop a computation
+  #
+  # See https://developers.signalfx.com/v2/reference#section-stop-a-computation
+  #
+  # @param reason [String] Reason for stopping the computation.
+  def stop(reason=nil)
+    @stop_func.call(@handle, reason)
+  end
+end
+
+class ComputationFailure < Exception
+end

data/lib/signalfx/signalflow/queue.rb

@@ -0,0 +1,43 @@
+require 'thread'
+
+# Borrowed from
+# https://spin.atomicobject.com/2017/06/28/queue-pop-with-timeout-fixed/
+
+class QueueWithTimeout
+  def initialize
+    @mutex = Mutex.new
+    @queue = []
+    @received = ConditionVariable.new
+  end
+ 
+  def <<(x)
+    @mutex.synchronize do
+      @queue << x
+      @received.signal
+    end
+  end
+ 
+  def pop(non_block = false)
+    pop_with_timeout(non_block ? 0 : nil)
+  end
+ 
+  def pop_with_timeout(timeout = nil)
+    @mutex.synchronize do
+      if timeout.nil?
+        # wait indefinitely until there is an element in the queue
+        while @queue.empty?
+          @received.wait(@mutex)
+        end
+      elsif @queue.empty? && timeout != 0
+        # wait for element or timeout
+        timeout_time = timeout + Time.now.to_f
+        while @queue.empty? && (remaining_time = timeout_time - Time.now.to_f) > 0
+          @received.wait(@mutex, remaining_time)
+        end
+      end
+      #if we're still empty after the timeout, raise exception
+      raise ThreadError, "queue empty" if @queue.empty?
+      @queue.shift
+    end
+  end
+end

data/lib/signalfx/signalflow/websocket.rb

@@ -0,0 +1,317 @@
+# Copyright (C) 2017 SignalFx, Inc. All rights reserved.
+
+require 'json'
+require 'thread'
+require 'websocket-client-simple'
+require 'eventmachine'
+
+require_relative './binary'
+require_relative './channel'
+require_relative './computation'
+
+
+# A WebSocket transport for SignalFlow.  This should not be used directly by
+# end-users.
+class SignalFlowWebsocketTransport
+  DETACHED = "DETACHED"
+
+  # A lower bound on the amount of time to wait for a computation to start
+  COMPUTATION_START_TIMEOUT_SECONDS = 30
+
+  def initialize(api_token, stream_endpoint)
+    @api_token = api_token
+    @stream_endpoint = stream_endpoint
+    @compress = true
+
+    @lock = Mutex.new
+    @close_reason = nil
+    reinit
+  end
+
+  def reinit
+    @ws = nil
+    @authenticated = false
+    @chan_callbacks = {}
+
+    name_lock = Mutex.new
+    num = 0
+    # Returns a unique channel name each time it is called
+    @channel_namer = ->{
+      name_lock.synchronize do
+        num += 1
+        "channel-#{num}"
+      end
+    }
+  end
+  private :reinit
+
+  # Starts a job (either execute or preflight) and waits until the JOB_START
+  # message is received with the computation handle arrives so that we can
+  # create a properly initialized computation object.  Yields to the given
+  # block which should send the WS message to start the job.
+  def start_job
+    computation = nil
+
+    channel = make_new_channel
+
+    yield channel.name
+
+    while true
+      begin
+        msg = channel.pop(COMPUTATION_START_TIMEOUT_SECONDS)
+      rescue ChannelTimeout
+        raise "Computation did not start after at least #{COMPUTATION_START_TIMEOUT_SECONDS} seconds"
+      end
+      if msg[:type] == "error"
+        raise ComputationFailure.new(msg[:message])
+      end
+
+      # STREAM_START comes before this but contains no useful information
+      if msg[:event] == "JOB_START"
+        computation = Computation.new(msg[:handle], method(:attach), method(:stop))
+        computation.channel = channel
+      elsif msg[:type] == "computation-started"
+        computation = Computation.new(msg[:computationId], method(:attach), method(:stop))
+        # Start jobs only use the channel to get error messages and can
+        # detach from the channel once the job has started.
+        channel.detach
+      else
+        next
+      end
+
+      return computation
+    end
+  end
+
+  def execute(program, start: nil, stop: nil, resolution: nil, max_delay: nil, persistent: nil)
+    start_job do |channel_name|
+      send_msg({
+        :type => "execute",
+        :channel => channel_name,
+        :program => program,
+        :start => start,
+        :stop => stop,
+        :resolution => resolution,
+        :max_delay => max_delay,
+        :persistent => persistent,
+        :compress => @compress,
+      }.reject!{|k,v| v.nil?}.to_json)
+    end
+  end
+
+  def preflight(program, start, stop, resolution: nil, max_delay: nil)
+    start_job do |channel_name|
+      send_msg({
+        :type => "preflight",
+        :channel => channel_name,
+        :program => program,
+        :start => start,
+        :stop => stop,
+        :resolution => resolution,
+        :max_delay => max_delay,
+        :compress => @compress,
+      }.reject!{|k,v| v.nil?}.to_json)
+    end
+  end
+
+  def start(program, start: nil, stop: nil, resolution: nil, max_delay: nil)
+    start_job do |channel_name|
+      send_msg({
+        :type => "start",
+        :channel => channel_name,
+        :program => program,
+        :start => start,
+        :stop => stop,
+        :resolution => resolution,
+        :max_delay => max_delay,
+      }.reject!{|k,v| v.nil?}.to_json)
+    end
+  end
+
+  def stop(handle, reason)
+    send_msg({
+      :type => "stop",
+      :handle => handle,
+      :reason => reason,
+    }.reject!{|k,v| v.nil?}.to_json)
+  end
+
+  # This doesn't actually work on the backend yet
+  def attach(handle, filters: nil, resolution: nil)
+    channel = make_new_channel
+
+    send_msg({
+      :type => "attach",
+      :channel => channel.name,
+      :handle => handle,
+      :filters => filters,
+      :resolution => resolution,
+      :compress => @compress,
+    }.reject!{|k,v| v.nil?}.to_json)
+
+    channel
+  end
+
+  def detach(channel, reason=nil)
+    send_msg({
+      :type => "detach",
+      :channel => channel,
+      :reason => reason,
+    }.to_json)
+
+    # There is no response message from the server signifying detach complete
+    # and there could be messages coming in even after the detach request is
+    # sent.  Therefore, use a sentinal value in place of the callback block so
+    # that the message receiver logic can distinguish this case from some
+    # anomolous case (say, due to bad logic in the code).
+    @chan_callbacks[channel] = DETACHED
+  end
+
+  def close
+    if @ws
+      @ws.close
+    end
+  end
+
+  def send_msg(msg)
+    @lock.synchronize do
+      if @ws.nil?
+        startup_client
+
+        # Polling is the simplest and most robust way to handle blocking until
+        # authenticated. Using ConditionVariable requires more complex logic
+        # that gains very little in terms of efficiecy given how quick auth
+        # should be.
+        start_time = Time.now
+        while !@authenticated
+          # The socket will be closed by the server if auth isn't successful
+          # within 5 seconds so no point in waiting longer
+          if Time.now - start_time > 5 || @close_reason
+            raise "Could not authenticate to SignalFlow WebSocket: #{@close_reason}"
+          end
+          sleep 0.1
+        end
+      end
+
+      @ws.send(msg)
+    end
+  end
+  private :send_msg
+
+  def on_close(msg)
+    @close_reason = "(#{msg.code}, #{msg.data})"
+    @chan_callbacks.keys.each do |channel_name|
+      invoke_callback_for_channel({ :event => "CONNECTION_CLOSED" }, channel_name)
+    end
+
+    reinit
+  end
+
+  def on_message(m)
+    begin
+      return if m.type == :ping
+      if m.type == :close
+        on_close(m)
+        return
+      end
+
+      message_received(m.data, m.type == :text)
+    rescue Exception => e
+      puts "Error processing SignalFlow message: #{e.backtrace.first}: #{e.message} (#{e.class})"
+    end
+  end
+
+  def on_open
+    @ws.send({
+      :type => "authenticate",
+      :token => @api_token,
+    }.to_json)
+  end
+
+  # Start up a new WS client in its own thread that runs an EventMachine
+  # reactor.
+  def startup_client
+    this = self
+    WebSocket::Client::Simple.connect("#{@stream_endpoint}/v2/signalflow/connect",
+                                      # Verification is disabled by default so this is essential
+                                      {verify_mode: OpenSSL::SSL::VERIFY_PEER}) do |ws|
+      @ws = ws
+      ws.on :error do |e|
+        puts "ERROR #{e.inspect}"
+      end
+
+      ws.on :close do |e|
+        this.on_close(e)
+      end
+
+      ws.on :message do |m|
+        this.on_message(m)
+      end
+
+      ws.on :open do
+        this.on_open
+      end
+    end
+  end
+  private :startup_client
+
+  def invoke_callback_for_channel(msg, channel_name)
+    chan = @chan_callbacks[channel_name]
+
+    raise "Callback for channel #{channel_name} is missing!" unless chan
+
+    if chan == DETACHED
+      return
+    else
+      chan.inject_message(msg)
+    end
+  end
+  private :invoke_callback_for_channel
+
+  def message_received(raw_msg, is_text)
+    msg = add_parsed_timestamp!(parse_message(raw_msg, is_text))
+
+    if msg[:type] == "authenticated"
+      @authenticated = true
+      return
+    end
+
+    if msg[:channel]
+      invoke_callback_for_channel(msg, msg[:channel])
+    else
+      # Ignore keep-alives
+      if msg[:event] == "KEEP_ALIVE"
+        return
+      else
+        raise "Unknown SignalFlow message: #{msg}"
+      end
+    end
+  end
+  private :message_received
+
+  def parse_message(raw_msg, is_text)
+    if is_text
+      JSON.parse(raw_msg, {:symbolize_names => true})
+    else
+      BinaryMessageParser.parse(raw_msg)
+    end
+  end
+  private :parse_message
+
+  def add_parsed_timestamp!(msg)
+    if msg.has_key?(:timestampMs)
+      msg[:timestamp] = Time.at(msg[:timestampMs] / 1000.0)
+    end
+    msg
+  end
+  private :add_parsed_timestamp!
+
+  def make_new_channel
+    name = @channel_namer.()
+    channel = Channel.new(name, ->(){ detach(name) })
+    @chan_callbacks[name] = channel
+    channel
+  end
+  private :make_new_channel
+end
+

data/lib/signalfx/version.rb

@@ -1,6 +1,6 @@
 # Copyright (C) 2015-2016 SignalFx, Inc. All rights reserved.
 
 module Version
-  VERSION = '1.0.2'
+  VERSION = '2.0.1'
   NAME = 'signalfx-ruby-client'
 end

data/signalfx.gemspec

@@ -10,7 +10,7 @@ Gem::Specification.new do |spec|
   spec.email         = ["info@signalfx.com"]
 
   spec.summary       = "Ruby client library for SignalFx"
-  spec.description   = "This is a programmatic interface in Ruby for SignalFx's metadata and ingest APIs. It is meant to provide a base for communicating with SignalFx APIs that can be easily leveraged by scripts and applications to interact with SignalFx or report metric and event data to SignalFx. Library supports Ruby 2.x versions"
+  spec.description   = "This is a programmatic interface in Ruby for SignalFx's metadata and ingest APIs. It is meant to provide a base for communicating with SignalFx APIs that can be easily leveraged by scripts and applications to interact with SignalFx or report metric and event data to SignalFx. Library supports Ruby 2.2.x+ versions"
   spec.homepage      = "https://signalfx.com"
   spec.license       = "Apache Software License v2 © SignalFx"
 
@@ -33,6 +33,10 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 10.0"
   spec.add_development_dependency "rspec", "~> 3.3"
   spec.add_development_dependency "webmock", "~> 2.1"
+  spec.add_development_dependency "thin", "~> 1.7"
+  spec.add_development_dependency "pry"
+  spec.add_development_dependency "faye-websocket", "~> 0.10.7"
   spec.add_dependency "protobuf", "~> 3.5.1", ">= 3.5.1"
   spec.add_dependency "rest-client", "~> 2.0"
+  spec.add_dependency 'websocket-client-simple', "~> 0.3.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: signalfx
 version: !ruby/object:Gem::Version
-  version: 1.0.2
+  version: 2.0.1
 platform: ruby
 authors:
 - SignalFx, Inc
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2016-09-19 00:00:00.000000000 Z
+date: 2017-09-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -66,6 +66,48 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.1'
+- !ruby/object:Gem::Dependency
+  name: thin
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.7'
+- !ruby/object:Gem::Dependency
+  name: pry
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: faye-websocket
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.7
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.10.7
 - !ruby/object:Gem::Dependency
   name: protobuf
   requirement: !ruby/object:Gem::Requirement
@@ -100,10 +142,24 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '2.0'
+- !ruby/object:Gem::Dependency
+  name: websocket-client-simple
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: 0.3.0
 description: This is a programmatic interface in Ruby for SignalFx's metadata and
   ingest APIs. It is meant to provide a base for communicating with SignalFx APIs
   that can be easily leveraged by scripts and applications to interact with SignalFx
-  or report metric and event data to SignalFx. Library supports Ruby 2.x versions
+  or report metric and event data to SignalFx. Library supports Ruby 2.2.x+ versions
 email:
 - info@signalfx.com
 executables: []
@@ -119,12 +175,19 @@ files:
 - bin/console
 - bin/setup
 - examples/generic_usecase.rb
+- examples/signalflow.rb
 - lib/proto/signal_fx_protocol_buffers.pb.rb
 - lib/signalfx.rb
 - lib/signalfx/conf.rb
 - lib/signalfx/json_signal_fx_client.rb
 - lib/signalfx/protobuf_signal_fx_client.rb
 - lib/signalfx/signal_fx_client.rb
+- lib/signalfx/signalflow/binary.rb
+- lib/signalfx/signalflow/channel.rb
+- lib/signalfx/signalflow/client.rb
+- lib/signalfx/signalflow/computation.rb
+- lib/signalfx/signalflow/queue.rb
+- lib/signalfx/signalflow/websocket.rb
 - lib/signalfx/version.rb
 - signalfx.gemspec
 homepage: https://signalfx.com
@@ -147,7 +210,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.6.10
 signing_key: 
 specification_version: 4
 summary: Ruby client library for SignalFx