ffwd-protobuf 0.1.6 → 0.1.7

data/lib/ffwd/plugin/protobuf/serializer/protocol0.rb

@@ -0,0 +1,121 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require 'ffwd/protocol0.pb'
+
+module FFWD
+  module Plugin
+    module Protobuf
+      module Serializer
+      end
+    end
+  end
+end
+
+module FFWD::Plugin::Protobuf::Serializer
+  module Protocol0
+    P = ::FFWD::Protocol0
+
+    def self.load string
+      message = P::Message.new
+      message.parse_from_string string
+
+      if message.has_field?(:event)
+        yield :event, receive_event(message.event)
+      end
+
+      if message.has_field?(:metric)
+        yield :metric, receive_metric(message.metric)
+      end
+    end
+
+    def self.receive_event e
+      d = {}
+      d[:time] = Time.at(e.time.to_f / 1000) if e.has_field?(:time)
+      d[:key] = e.key if e.has_field?(:key)
+      d[:value] = e.value if e.has_field?(:value)
+      d[:host] = e.host if e.has_field?(:host)
+      d[:state] = e.state if e.has_field?(:state)
+      d[:description] = e.description if e.has_field?(:description)
+      d[:ttl] = e.ttl if e.has_field?(:ttl)
+      d[:tags] = from_tags e.tags if e.tags
+      d[:attributes] = from_attributes e.attributes if e.attributes
+      return d
+    end
+
+    def self.receive_metric m
+      d = {}
+      d[:proc] = m.host if m.has_field?(:proc)
+      d[:time] = Time.at(m.time.to_f / 1000) if m.has_field?(:time)
+      d[:key] = m.key if m.has_field?(:key)
+      d[:value] = m.value if m.has_field?(:value)
+      d[:host] = m.host if m.has_field?(:host)
+      d[:tags] = from_tags m.tags if m.tags
+      d[:attributes] = from_attributes m.attributes if m.attributes
+      return d
+    end
+
+    def self.from_attributes attributes
+      Hash[attributes.map{|a| [a.key, a.value]}]
+    end
+
+    def self.from_tags tags
+      Array.new tags
+    end
+
+    def self.dump_event event
+      e = P::Event.new
+      e.time = (event.time.to_f * 1000).to_i if event.time
+      e.key = event.key if event.key
+      e.value = event.value.to_f if event.value
+      e.host = event.host if event.host
+      e.state = event.state if event.state
+      e.description = event.description if event.description
+      e.ttl = event.ttl if event.ttl
+      e.tags = to_tags event.tags if event.tags
+      e.attributes = to_attributes event.attributes if event.attributes
+
+      message = P::Message.new
+      message.event = e
+      message.serialize_to_string
+    end
+
+    def self.dump_metric metric
+      m = P::Metric.new
+      m.time = (metric.time.to_f * 1000).to_i if metric.time
+      m.key = metric.key if metric.key
+      m.value = metric.value.to_f if metric.value
+      m.host = metric.host if metric.host
+      m.tags = to_tags metric.tags if metric.tags
+      m.attributes = to_attributes metric.attributes if metric.attributes
+
+      message = P::Message.new
+      message.metric = m
+      message.serialize_to_string
+    end
+
+    private
+
+    def self.to_attributes attributes
+      attributes.map do |key, value|
+        P::Attribute.new(:key => key.to_s, :value => value.to_s)
+      end
+    end
+
+    def self.to_tags tags
+      Array.new tags
+    end
+  end
+end

data/lib/ffwd/plugin/protobuf/serializer.rb

@@ -0,0 +1,71 @@
+# $LICENSE
+# Copyright 2013-2014 Spotify AB. All rights reserved.
+#
+# The contents of this file are licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with the
+# License. You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+
+require_relative 'serializer/protocol0'
+
+module FFWD::Plugin::Protobuf::Serializer
+  VERSIONS = {
+    0 => Protocol0
+  }
+
+  LATEST_VERSION = 0
+
+  def self.dump_frame string
+    [LATEST_VERSION, string.length + 8].pack("NN") + string
+  end
+
+  def self.load_frame string
+    if string.length < 8
+      raise "Frame too small, expected at least 8 bytes but got #{string.length}"
+    end
+
+    version = string[0..3].unpack("N")[0]
+    length = string[4..7].unpack("N")[0]
+
+    unless impl = VERSIONS[version]
+      raise "Unsupported protocol version #{version}, latest is #{LATEST_VERSION}"
+    end
+
+    if length != string.length
+      raise "Message length invalid, expected #{length} but got #{string.length}"
+    end
+
+    [impl, string[8..length]]
+  end
+
+  def self.dump_event event
+    unless impl = VERSIONS[LATEST_VERSION]
+      raise "No implementation for latest version: #{LATEST_VERSION}"
+    end
+
+    dump_frame impl.dump_event event
+  end
+
+  def self.dump_metric metric
+    unless impl = VERSIONS[LATEST_VERSION]
+      raise "No implementation for latest version: #{LATEST_VERSION}"
+    end
+
+    dump_frame impl.dump_metric metric
+  end
+
+  def self.load string
+    impl, string = load_frame string
+
+    impl.load string do |type, data|
+      yield type, data
+    end
+  end
+end

data/lib/ffwd/plugin/protobuf/version.rb

@@ -16,7 +16,7 @@
 module FFWD
   module Plugin
     module Protobuf
-      VERSION = "0.1.6"
+      VERSION = "0.1.7"
     end
   end
 end

data/lib/ffwd/plugin/protobuf.rb

@@ -24,38 +24,17 @@ require 'ffwd/protocol'
 require 'ffwd/event'
 require 'ffwd/metric'
 
-require_relative 'protobuf/protocol.pb'
+require_relative 'protobuf/serializer'
 
 module FFWD::Plugin::Protobuf
   include FFWD::Plugin
   include FFWD::Logging
 
-  P = ::FFWD::Plugin::Protobuf::Protocol
-
   register_plugin "protobuf"
 
-  module Serializer
-    def self.dump m
-      m.serialize_to_string
-    end
-
-    def self.load string
-      m = P::Message.new
-      m.parse_from_string string
-      return m
-    end
-  end
-
-  class Output < FFWD::Handler
-    include EM::Protocols::ObjectProtocol
-    include Serializer
-
+  class OutputUDP < FFWD::Handler
     def self.plugin_type
-      "protobuf_out"
-    end
-
-    def serializer
-      Serializer
+      "protobuf_udp_out"
     end
 
     def send_all events, metrics
@@ -69,56 +48,15 @@ module FFWD::Plugin::Protobuf
     end
 
     def send_event event
-      message = P::Message.new
-      e = P::Event.new
-      e.time = (event.time.to_f * 1000).to_i if event.time
-      e.key = event.key if event.key
-      e.value = to_value event.value if event.value
-      e.host = event.host if event.host
-      e.source = event.source if event.source
-      e.state = event.state if event.state
-      e.description = event.description if event.description
-      e.ttl = event.ttl if event.ttl
-      e.tags = to_tags event.tags if event.tags
-      e.attributes = to_attributes event.attributes if event.attributes
-      message.event = e
-      send_object message
+      parent.send_data Serializer.dump_event(event)
     end
 
     def send_metric metric
-      message = P::Message.new
-      m = P::Metric.new
-      m.time = (metric.time.to_f * 1000).to_i if metric.time
-      m.key = metric.key if metric.key
-      m.value = to_value metric.value if metric.value
-      m.host = metric.host if metric.host
-      m.source = metric.source if metric.source
-      m.tags = to_tags metric.tags if metric.tags
-      m.attributes = to_attributes metric.attributes if metric.attributes
-      message.metric = m
-      send_object message
-    end
-
-    private
-
-    def to_value value
-      v = P::Value.new
-      v.value_d = value
-      return v
-    end
-
-    def to_attributes attributes
-      attributes.map do |key, value|
-        P::Attribute.new(:key => key, :value => value)
-      end
-    end
-
-    def to_tags tags
-      Array.new tags
+      parent.send_data Serializer.dump_metric(metric)
     end
   end
 
-  class Input < FFWD::Connection
+  class InputUDP < FFWD::Connection
     include FFWD::Logging
     include EM::Protocols::FrameObjectProtocol
 
@@ -132,79 +70,35 @@ module FFWD::Plugin::Protobuf
       "protobuf_in"
     end
 
-    def serializer
-      Serializer
-    end
-
-    def receive_object message
-      if message.has_field?(:event)
-        receive_event message.event
-      end
+    def receive_data datagram
+      Serializer.load(datagram) do |type, data|
+        if type == :event
+          @core.input.event data
+          @bind.increment :received_events
+          next
+        end
 
-      if message.has_field?(:metric)
-        receive_metric message.metric
+        if type == :metric
+          @core.input.metric data
+          @bind.increment :received_metrics
+          next
+        end
       end
-    end
+    rescue => e
+      @log.error "Failed to receive data", e
 
-    def handle_exception datagram, e
-      @log.error("Failed to decode protobuf object", e)
       if @log.debug?
-        @log.debug("FRAME: " + FFWD.dump2hex(datagram))
+        @log.debug("DUMP: " + FFWD.dump2hex(datagram))
       end
     end
-
-    def receive_event e
-      d = {}
-      d[:time] = Time.at(e.time.to_f / 1000) if e.has_field?(:time)
-      d[:key] = e.key if e.has_field?(:key)
-      d[:value] = e.value if e.has_field?(:value)
-      d[:host] = e.host if e.has_field?(:host)
-      d[:source] = e.source if e.has_field?(:source)
-      d[:state] = e.state if e.has_field?(:state)
-      d[:description] = e.description if e.has_field?(:description)
-      d[:ttl] = e.ttl if e.has_field?(:ttl)
-      d[:tags] = from_tags m.tags if m.tags
-      d[:attributes] = from_attributes m.attributes if m.attributes
-      @core.input.event d
-      @bind.increment :received_events
-    rescue => e
-      @log.error "Failed to receive event", e
-      @bind.increment :failed_events
-    end
-
-    def receive_metric m
-      d = {}
-      d[:time] = Time.at(m.time.to_f / 1000) if m.has_field?(:time)
-      d[:key] = m.key if m.has_field?(:key)
-      d[:value] = m.value if m.has_field?(:value)
-      d[:host] = m.host if m.has_field?(:host)
-      d[:source] = m.source if m.has_field?(:source)
-      d[:tags] = from_tags m.tags if m.tags
-      d[:attributes] = from_attributes m.attributes if m.attributes
-      @core.input.metric d
-      @bind.increment :received_metrics
-    rescue => e
-      @log.error "Failed to receive event", e
-      @bind.increment :failed_metrics
-    end
-
-    private
-
-    def from_attributes attributes
-      Hash[attributes.map{|a| [a.key, a.value]}]
-    end
-
-    def from_tags tags
-      Array.new tags
-    end
   end
 
   DEFAULT_HOST = "localhost"
   DEFAULT_PORT = 19091
   DEFAULT_PROTOCOL = 'udp'
 
-  OUTPUTS = {:udp => Output}
-  INPUTS = {:udp => Input}
+  OUTPUTS = {:udp => OutputUDP}
+  INPUTS = {:udp => InputUDP}
 
   def self.setup_output opts, core
     opts[:host] ||= DEFAULT_HOST
@@ -212,11 +106,11 @@ module FFWD::Plugin::Protobuf
 
     protocol = FFWD.parse_protocol(opts[:protocol] || DEFAULT_PROTOCOL)
 
-    unless type = OUTPUTS[protocol.family]
+    unless handler = OUTPUTS[protocol.family]
       raise "No type for protocol family: #{protocol.family}"
     end
 
-    protocol.connect opts, core, log, type
+    protocol.connect opts, core, log, handler
   end
 
   def self.setup_input opts, core

data/lib/ffwd/protocol0.pb.rb

@@ -0,0 +1,93 @@
+### Generated by rprotoc. DO NOT EDIT!
+### <proto file: ./plugins/ffwd-protobuf/proto/protocol0.proto>
+# package FFWD.Protocol0;
+# 
+# message Metric {
+#   // processor to use for metric.
+#   optional string proc = 1;
+#   // time in ms when metric was generated.
+#   optional int64 time = 2;
+#   // key of metric.
+#   optional string key = 3;
+#   // value of metric.
+#   optional double value = 4;
+#   // host where metric originated.
+#   optional string host = 5;
+#   // tags associated to metric.
+#   repeated string tags = 6;
+#   // attributes associated to metric.
+#   repeated Attribute attributes = 7;
+# }
+# 
+# message Event {
+#   // time in ms when the event was generated.
+#   optional int64 time = 1;
+#   // key of event.
+#   optional string key = 2;
+#   // value of event.
+#   optional double value = 3;
+#   // host where event originated.
+#   optional string host = 4;
+#   // indicated state of this event.
+#   optional string state = 5;
+#   // description of event.
+#   optional string description = 6;
+#   // time this event should be considered valid in seconds.
+#   optional int64 ttl = 7;
+#   // tags associated to event.
+#   repeated string tags = 8;
+#   // attributes associated to event.
+#   repeated Attribute attributes = 9;
+# }
+# 
+# message Attribute {
+#   required string key = 1;
+#   optional string value = 2;
+# }
+# 
+# message Message {
+#   optional Metric metric = 1;
+#   optional Event event = 2;
+# }
+
+require 'protobuf/message/message'
+require 'protobuf/message/enum'
+require 'protobuf/message/service'
+require 'protobuf/message/extend'
+
+module FFWD
+  module Protocol0
+    class Metric < ::Protobuf::Message
+      defined_in __FILE__
+      optional :string, :proc, 1
+      optional :int64, :time, 2
+      optional :string, :key, 3
+      optional :double, :value, 4
+      optional :string, :host, 5
+      repeated :string, :tags, 6
+      repeated :Attribute, :attributes, 7
+    end
+    class Event < ::Protobuf::Message
+      defined_in __FILE__
+      optional :int64, :time, 1
+      optional :string, :key, 2
+      optional :double, :value, 3
+      optional :string, :host, 4
+      optional :string, :state, 5
+      optional :string, :description, 6
+      optional :int64, :ttl, 7
+      repeated :string, :tags, 8
+      repeated :Attribute, :attributes, 9
+    end
+    class Attribute < ::Protobuf::Message
+      defined_in __FILE__
+      required :string, :key, 1
+      optional :string, :value, 2
+    end
+    class Message < ::Protobuf::Message
+      defined_in __FILE__
+      optional :Metric, :metric, 1
+      optional :Event, :event, 2
+    end
+  end
+end

metadata

@@ -1,18 +1,20 @@
 --- !ruby/object:Gem::Specification
 name: ffwd-protobuf
 version: !ruby/object:Gem::Version
-  version: 0.1.6
+  version: 0.1.7
+  prerelease: 
 platform: ruby
 authors:
 - John-John Tedro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-29 00:00:00.000000000 Z
+date: 2014-05-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ruby_protobuf
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -20,6 +22,7 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -27,20 +30,23 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: ffwd
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.7
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
-    - - ! '>='
+    - - '='
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 0.1.7
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -48,6 +54,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -55,6 +62,7 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: rspec-mocks
   requirement: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -62,6 +70,7 @@ dependencies:
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
+    none: false
     requirements:
     - - ! '>='
       - !ruby/object:Gem::Version
@@ -73,31 +82,34 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- lib/ffwd/plugin/protobuf.rb
+- lib/ffwd/protocol0.pb.rb
 - lib/ffwd/plugin/protobuf/version.rb
-- lib/ffwd/plugin/protobuf/protocol.pb.rb
+- lib/ffwd/plugin/protobuf/serializer/protocol0.rb
+- lib/ffwd/plugin/protobuf/serializer.rb
+- lib/ffwd/plugin/protobuf.rb
 homepage: https://github.com/spotify/ffwd
 licenses:
 - Apache 2.0
-metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
   requirements:
   - - ! '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.3
+rubygems_version: 1.8.23
 signing_key: 
-specification_version: 4
+specification_version: 3
 summary: protobuf reference protocol support for FFWD.
 test_files: []

checksums.yaml

@@ -1,15 +0,0 @@
----
-!binary "U0hBMQ==":
-  metadata.gz: !binary |-
-    NTFmZTlmNTgzODA5NGQ3Yjc0Y2M0ZTE4OTFlNWI4ZTFlZWU1NzQyZg==
-  data.tar.gz: !binary |-
-    ZTM4MTYxMWI1MzgwY2Q5NDAxMzFjNGJkZTkzNTQ5NTAwY2JlODVkNQ==
-!binary "U0hBNTEy":
-  metadata.gz: !binary |-
-    ZjBhZDYxZGZjN2ExY2IwZWY1NDFjODk1MDQwN2NlNjM5MmMyODYwY2IyYjk1
-    YjU4ZDU3NjY2ZTljMmQ0OTQ2NjJmYjY2MmM1OGJiY2Y4NWY1ZmNmNzViYWVh
-    NzQ2MzlmMjA1NDg4MDc4MDYzMWM3YTIwNzcwODc3MmY3MTk3ZTg=
-  data.tar.gz: !binary |-
-    YzU3MTY2Y2Q5MmNmMmUyYTA0Y2U4NWI4NzllMDIzMTRiOTgxNTRhYmM4OTBm
-    NWY0ZjZjM2FlMGU1NWFiNjI4MDkxYjZkMTk5ZjRmMTU0MmE4N2E4NGM3Mzhk
-    MWE4YjY3NTcwYTg0MjliYjgxMDJjMmQ3OTVjMWIwZjUxMThhMDM=

data/lib/ffwd/plugin/protobuf/protocol.pb.rb

@@ -1,105 +0,0 @@
-### Generated by rprotoc. DO NOT EDIT!
-### <proto file: ./proto/protocol.proto>
-# package FFWD.Plugin.Protobuf.Protocol;
-# 
-# option java_package = "com.spotify.ffwd";
-# option java_outer_classname = "Protocol";
-# 
-# message Metric {
-#   // time in ms when metric was generated.
-#   optional int64 time = 1;
-#   // key of metric.
-#   optional string key = 2;
-#   // value of metric.
-#   optional double value = 3;
-#   // host where metric originated.
-#   optional string host = 4;
-#   // source metric of the metric.
-#   optional string source = 5;
-#   // tags associated to metric.
-#   repeated string tags = 6;
-#   // attributes associated to metric.
-#   repeated Attribute attributes = 7;
-# }
-# 
-# message Event {
-#   // time in ms when the event was generated.
-#   optional int64 time = 1;
-#   // key of event.
-#   optional string key = 2;
-#   // value of event.
-#   optional double value = 3;
-#   // host where event originated.
-#   optional string host = 4;
-#   // source event (if any).
-#   optional string source = 5;
-#   // indicated state of this event.
-#   optional string state = 6;
-#   // description of event.
-#   optional string description = 7;
-#   // time this event should be considered valid in seconds.
-#   optional int64 ttl = 8;
-#   // tags associated to event.
-#   repeated string tags = 9;
-#   // attributes associated to event.
-#   repeated Attribute attributes = 10;
-# }
-# 
-# message Attribute {
-#   required string key = 1;
-#   optional string value = 2;
-# }
-# 
-# message Message {
-#   optional Metric metric = 1;
-#   optional Event event = 2;
-# }
-
-require 'protobuf/message/message'
-require 'protobuf/message/enum'
-require 'protobuf/message/service'
-require 'protobuf/message/extend'
-
-module FFWD
-  module Plugin
-    module Protobuf
-      module Protocol
-        ::Protobuf::OPTIONS[:"java_package"] = "com.spotify.ffwd"
-        ::Protobuf::OPTIONS[:"java_outer_classname"] = "Protocol"
-        class Metric < ::Protobuf::Message
-          defined_in __FILE__
-          optional :int64, :time, 1
-          optional :string, :key, 2
-          optional :double, :value, 3
-          optional :string, :host, 4
-          optional :string, :source, 5
-          repeated :string, :tags, 6
-          repeated :Attribute, :attributes, 7
-        end
-        class Event < ::Protobuf::Message
-          defined_in __FILE__
-          optional :int64, :time, 1
-          optional :string, :key, 2
-          optional :double, :value, 3
-          optional :string, :host, 4
-          optional :string, :source, 5
-          optional :string, :state, 6
-          optional :string, :description, 7
-          optional :int64, :ttl, 8
-          repeated :string, :tags, 9
-          repeated :Attribute, :attributes, 10
-        end
-        class Attribute < ::Protobuf::Message
-          defined_in __FILE__
-          required :string, :key, 1
-          optional :string, :value, 2
-        end
-        class Message < ::Protobuf::Message
-          defined_in __FILE__
-          optional :Metric, :metric, 1
-          optional :Event, :event, 2
-        end
-      end
-    end
-  end
-end