vayacondios-server 0.1.2 → 0.1.6

data/Gemfile

@@ -9,4 +9,5 @@ group :hadoop_monitor do
   gem 'json'
   gem 'nokogiri', '~> 1.5'
   gem 'nibbler',  '~> 1.3'
+  gem "swineherd-fs", "~> 0.0.3"
 end

data/app/http_shim.rb

@@ -3,12 +3,8 @@
 require 'vayacondios-server'
 
 class HttpShim < Goliath::API
+  use Vayacondios::Rack::AssumeJSON                                      # assume application/json content type
   use Goliath::Rack::Tracer, 'X-Tracer'                                  # log trace statistics
-  # /v1/infochimps/itemset/foo/1 -d '["foo","bar","baz"]'
-  # use Vayacondios::Rack::Versionator
-  # /v1/infochimps/itemset/foo/1 -d '{"items":["foo","bar","baz"]}' ... env[:vayacondios_version] = 1
-  # post_process()
-  # extract results = results[:results] from body for legacy
   use Goliath::Rack::Params                                              # parse query string and message body into params hash
   use Goliath::Rack::Validation::RequestMethod, %w[GET PUT PATCH DELETE] # only allow these methods
   use Vayacondios::Rack::ExtractMethods                                  # interpolate GET, PUT into :create, :update, etc

data/lib/vayacondios-client.rb

@@ -16,5 +16,7 @@ require 'gorillib/string/constantize'
 require 'gorillib/string/inflections'
 
 require 'vayacondios/client/http_client'
+require 'vayacondios/client/cube_client'
+require 'vayacondios/client/zabbix_client'
 require 'vayacondios/client/notifier'
 require 'vayacondios/client/configliere'

data/lib/vayacondios-server.rb

@@ -22,6 +22,7 @@ require 'vayacondios/server/handlers/config_handler'
 require 'vayacondios/server/handlers/event_handler'
 require 'vayacondios/server/handlers/itemset_handler'
 
+require 'vayacondios/server/rack/assume_json'
 require 'vayacondios/server/rack/extract_methods'
 require 'vayacondios/server/rack/path'
 require 'vayacondios/server/rack/path_validation'

data/lib/vayacondios/client/cube_client.rb

@@ -0,0 +1,39 @@
+class Vayacondios
+  class CubeClient
+    include Gorillib::Model
+
+    field :host,         String,  :default => 'localhost'
+    field :port,         Integer, :default => 6000
+    
+    class Error < StandardError; end
+
+    def uri
+      return @uri if @uri
+
+      uri_str  = "http://#{host}:#{port}/1.0"
+      @uri ||= URI(uri_str)
+    end
+    
+    def event(topic, document = {})
+      request(:post, File.join(uri.path, 'event'), MultiJson.dump(document))
+    end
+    
+  private
+    
+    def request(method, path, document=nil)
+      http    = Net::HTTP.new(uri.host, uri.port)
+
+      params  = [method.to_sym, path]
+      params += [document, {'Content-Type' => 'application/json'}] unless document.nil?
+      
+      response = http.send *params
+
+      if Net::HTTPSuccess === response
+        MultiJson.load(response.body) rescue response.body
+      else
+        raise Error.new("Error (#{response.code}) while #{method.to_s == 'get' ? 'fetching' : 'inserting'} document: " + response.body)
+      end
+    end
+  end
+end
+  

data/lib/vayacondios/client/itemset.rb

@@ -4,42 +4,29 @@ require 'multi_json'
 class Vayacondios
   class Client
     class ItemSet
-      def initialize host, port, organization, topic, id
-        @host = host
-        @port = port
-
-        @path = "/v1/#{organization}/itemset/#{topic}/#{id}"
+      def initialize host, port, organization=nil, topic=nil, id=nil
+        @host         = host
+        @port         = port
+        @organization = organization
+        @topic        = topic
+        @id           = id
       end
 
-      # Exposed for testing.
-      def _req type, ary = nil
-        case type
-        when :fetch  then
-          req = Net::HTTP::Get.new(@path)
-        when :create then
-          (req = Net::HTTP::Put.new(@path)).body = MultiJson.encode(ary)
-        when :update then
-          (req = Net::HTTP::Put.new(@path, {"x-method" => "PATCH"})).body = MultiJson.encode(ary)
-        when :remove then
-          (req = Net::HTTP::Delete.new(@path)).body = MultiJson.encode(ary)
-        end
-        req
+      def fetch organization=nil, topic=nil, id=nil
+        resp = execute_request(_req(:fetch, nil, organization, topic, id)) and
+          resp["contents"]
       end
 
-      def fetch
-        execute_request(_req(:fetch))
+      def update ary, organization=nil, topic=nil, id=nil
+        execute_request(_req(:update, ary, organization, topic, id))
       end
 
-      def update ary
-        execute_request(_req(:update, ary))
+      def create ary, organization=nil, topic=nil, id=nil
+        execute_request(_req(:create, ary, organization, topic, id))
       end
 
-      def create ary
-        execute_request(_req(:create, ary))
-      end
-
-      def remove ary
-        execute_request(_req(:remove, ary))
+      def remove ary, organization=nil, topic=nil, id=nil
+        execute_request(_req(:remove, ary, organization, topic, id))
       end
 
 
@@ -52,6 +39,34 @@ class Vayacondios
         result = MultiJson.decode(resp) unless resp.nil? or resp.empty?
         (result.respond_to?(:has_key?) and result.has_key? "error") ? nil : result
       end
+
+      def path organization, topic, id
+        if ((the_organization = (organization || @organization)).nil? ||
+            (the_topic        = (topic        || @topic       )).nil? ||
+            (the_id           = (id           || @id          )).nil?)
+          raise ArgumentError.new("must provide organization, topic, and id!")
+        end
+
+        ['/v1', the_organization, 'itemset', the_topic, the_id].join("/")
+      end
+
+      # This is the only private method that is tested.
+      def _req type, ary=nil, organization=nil, topic=nil, id=nil
+
+        the_path = path(organization, topic, id)
+        headers = {"content-type" => "application/json"}
+        headers.merge!("x-method" => "PATCH") if type == :update
+ 
+        case type
+        when :fetch  then Net::HTTP::Get
+        when :create then Net::HTTP::Put
+        when :update then Net::HTTP::Put
+        when :remove then Net::HTTP::Delete
+        else         raise ArgumentError.new("invalid type: #{type}")
+        end.new(the_path, headers).tap do |req|
+          req.body = MultiJson.encode(contents: ary) unless type == :fetch
+        end
+      end
     end
   end
 end

data/lib/vayacondios/client/notifier.rb

@@ -56,11 +56,34 @@ class Vayacondios
     end
   end
 
+  class CubeNotifier < Notifier
+    def initialize(options={})
+      @client = Vayacondios::CubeClient.receive(options)
+    end
+    def notify topic, cargo={}
+      prepped = prepare(cargo)
+      client.event(topic, prepped)
+      nil
+    end
+  end
+
+  class ZabbixNotifier < Notifier
+    def initialize options={}
+      @client = Vayacondios::ZabbixClient.receive(options)
+    end
+    def notify(topic, cargo={})
+      prepped = prepare(cargo)
+      client.insert(topic, prepped)
+    end
+  end
+  
   class NotifierFactory
     def self.receive(attrs = {})
       type = attrs[:type]
       case type
       when 'http'        then HttpNotifier.new(attrs)
+      when 'cube'        then CubeNotifier.new(attrs)
+      when 'zabbix'      then ZabbixNotifier.new(attrs)
       when 'log'         then LogNotifier.new(attrs)
       when 'none','null' then NullNotifier.new(attrs)
       else
@@ -80,7 +103,7 @@ class Vayacondios
     def self.included klass
       if klass.ancestors.include? Gorillib::Model
         klass.class_eval do
-          field :notifier, Vayacondios::NotifierFactory, default: Vayacondios.default_notifier
+          field :notifier, Vayacondios::NotifierFactory, default: Vayacondios.default_notifier, :doc => "Notifier used to notify out of band data"
           
           def receive_notifier params
             params.merge!(log: try(:log)) if params[:type] == 'log'

data/lib/vayacondios/client/zabbix_client.rb

@@ -0,0 +1,148 @@
+require 'socket'
+
+class Vayacondios
+
+  # Used for sending events to a Zabbix server.
+  #
+  # An 'event' from Vayacondios' perspective is an arbitrary Hash.
+  #
+  # An 'event' from Zabbix's perspective is a tuple of values:
+  #
+  # * time
+  # * host
+  # * key
+  # * value
+  #
+  # This client will accept a Vayacondios event and internally
+  # translate it into a set of Zabbix events.
+  #
+  # @example A CPU monitoring notification
+  #
+  #   notify "foo-server.example.com", cpu: {
+  #     util: {
+  #       user: 0.20,
+  #       idle: 0.70,
+  #       sys:  0.10
+  #     },
+  #     load: 1.3
+  #   }
+  #
+  # would get turned into the following events when written to Zabbix:
+  #
+  # @example The CPU monitoring notification translated to Zabbix events
+  #
+  # [
+  #   { host: "foo-server.example.com", key: "cpu.util.user", value: 0.20 }
+  #   { host: "foo-server.example.com", key: "cpu.util.idle", value: 0.70 },
+  #   { host: "foo-server.example.com", key: "cpu.util.sys",  value: 0.10 },
+  #   { host: "foo-server.example.com", key: "cpu.load",      value: 1.3  }
+  # ]
+  #
+  # Zabbix will interpret the time as the time it receives each event.
+  #
+  # The following links provide details on the protocol used by Zabbix
+  # to receive events:
+  #
+  # * https://www.zabbix.com/forum/showthread.php?t=20047&highlight=sender
+  # * https://gist.github.com/1170577
+  # * http://spin.atomicobject.com/2012/10/30/collecting-metrics-from-ruby-processes-using-zabbix-trappers/?utm_source=rubyflow&utm_medium=ao&utm_campaign=collecting-metrics-zabix
+  class ZabbixClient
+    include Gorillib::Builder
+
+    attr_accessor :socket
+
+    field :host, String,  :default => 'localhost', :doc => "Host for the Zabbix server"
+    field :port, Integer, :default => 10051,       :doc => "Port for the Zabbix server"
+
+    # Insert events to a Zabbix server.
+    #
+    # The `topic` will be used as the name of the Zabbix host to
+    # associate event data to.
+    #
+    # As per the documentation for the [Zabbix sender
+    # protocol](https://www.zabbix.com/wiki/doc/tech/proto/zabbixsenderprotocol),
+    # a new TCP connection will be created for each event.
+    #
+    # @param [String] topic
+    # @param [Hash] cargo
+    # Array<Hash>] text
+    def insert topic, cargo={}
+      self.socket = TCPSocket.new(host, port)
+      send_request(topic, cargo)
+      handle_response
+      self.socket.close
+    end
+
+    private
+
+    # :nodoc
+    def send_request topic, cargo
+      socket.write(payload(topic, cargo))
+    end
+
+    # :nodoc
+    def handle_response
+      header = socket.recv(5)
+      if header == "ZBXD\1"
+        data_header = socket.recv(8)
+        length      = data_header[0,4].unpack("i")[0]
+        response    = MultiJson.load(socket.recv(length))
+        puts response["info"]
+      else
+        puts "Invalid response: #{header}"
+      end
+    end
+
+    # :nodoc
+    def payload topic, cargo={}
+      body = body_for(topic, cargo)
+      header_for(body) + body
+    end
+
+    # :nodoc
+    def body_for topic, cargo={}
+      MultiJson.dump({request: "sender data", data: zabbix_events_from(topic, cargo) })
+    end
+
+    # :nodoc
+    def header_for body
+      length = body.bytesize
+      "ZBXD\1".encode("ascii") + [length].pack("i") + "\x00\x00\x00\x00"
+    end
+
+    # :nodoc
+    def zabbix_events_from topic, cargo, scope=''
+      events = []
+      case cargo
+      when Hash
+        cargo.each_pair do |key, value|
+          events += zabbix_events_from(topic, value, new_scope(scope, key))
+        end
+      when Array
+        cargo.each_with_index do |item, index|
+          events += zabbix_events_from(topic, item, new_scope(scope, index))
+        end
+      else
+        events << event_body(topic, scope, cargo)
+      end
+      events
+    end
+
+    # :nodoc
+    def new_scope(current_scope, new_scope)
+      [current_scope, new_scope].map(&:to_s).reject(&:empty?).join('.')
+    end
+
+    # :nodoc
+    def event_body topic, scope, cargo
+      value = case cargo
+      when Hash  then cargo[:value]
+      when Array then cargo.first
+      else cargo
+      end
+      { host: topic, key: scope, value: value }
+    end
+    
+  end
+end
+

data/lib/vayacondios/server/handlers/itemset_handler.rb

@@ -41,7 +41,6 @@ class Vayacondios
       validate_options options
       
       existing_document = ItemsetDocument.find(@mongo, options)
-      puts "destroy existing"
       existing_document.destroy(document)
     end
 

data/lib/vayacondios/server/model/itemset_document.rb

@@ -10,7 +10,7 @@ require 'vayacondios/server/model/document'
 # work while Goliath is in a streaming context.
 
 class Vayacondios::ItemsetDocument < Vayacondios::Document
-  attr_reader :organization, :topic, :body, :id
+  attr_reader :organization, :topic, :id
 
   def initialize(mongodb, options = {})
     super options
@@ -33,13 +33,17 @@ class Vayacondios::ItemsetDocument < Vayacondios::Document
     self.new(mongodb, options).find
   end
 
+  def body
+    {contents: @body}
+  end
+
   def find
     result = @collection.find_one({_id: @id})
 
     if result
       result.delete("_id")
       @body = result["d"]
-      self 
+      self
     else
       nil
     end
@@ -48,7 +52,7 @@ class Vayacondios::ItemsetDocument < Vayacondios::Document
   def update(document)
     raise Vayacondios::Error::BadRequest.new unless document.is_a?(Hash)
 
-    @body = document['contents'] # should be items
+    @body = document['contents']
 
 
     @collection.update({:_id => @id}, {:_id => @id, 'd' => @body }, {upsert: true})
@@ -61,7 +65,7 @@ class Vayacondios::ItemsetDocument < Vayacondios::Document
 
     # Merge ourselves
     if @body
-      @body = body + document['contents']
+      @body = @body + document['contents']
     else
       @body = document['contents']
     end

data/lib/vayacondios/server/rack/assume_json.rb

@@ -0,0 +1,13 @@
+class Vayacondios
+  module Rack
+    class AssumeJSON
+      include Goliath::Rack::AsyncMiddleware
+      
+      def call(env)
+        env['CONTENT_TYPE'] =
+          'application/json' unless env.has_key? 'CONTENT_TYPE'
+        super env
+      end
+    end
+  end
+end

data/lib/vayacondios/server/rack/extract_methods.rb

@@ -12,7 +12,17 @@ class Vayacondios
         return unless env['REQUEST_METHOD']
         case env['REQUEST_METHOD'].upcase
         when 'PUT'    then
-          (env['HTTP_X_METHOD'] && env['HTTP_X_METHOD'].upcase == 'PATCH') ? :patch : :update
+          if env.has_key? 'HTTP_X_METHOD'
+            if env['HTTP_X_METHOD'].upcase == 'PATCH'
+              :patch
+            elsif env['HTTP_X_METHOD'].upcase == 'DELETE'
+              :delete
+            else
+              :update
+            end
+          else
+            :update
+          end
         when 'GET'    then :show
         when 'POST'   then :create
         when 'PATCH'  then :patch