ffwd-google-cloud 0.4.0 → 0.4.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: c105cd9903f52b23602bfb3b6c21aa0a25e02294
-  data.tar.gz: 41cc39f9192c0301fb2c450e9b5c214b4254b824
+  metadata.gz: 67ce27fdf5b4be025672e1f9976f65b8f0cf6b20
+  data.tar.gz: 73692e2a3c6ed2f31cd7a1a282d447ecb122cc57
 SHA512:
-  metadata.gz: 2b4cb5bdf6522e2e2b3a318fd57077f5962e7a3d8666ecd847e5abb6d41f03080466ad45532aa0844f5be38847c4511c44eb7309a94ab9f7c3931b4811abd729
-  data.tar.gz: 48ed69dc4005e1d82935c73894210ded2df4071136b682ee4d42ef6f16db4cc9e5482a72e7421b91774dba56460138d92f3f8bbeb2b1b041c7dc42738c5cee7a
+  metadata.gz: 05ee2f2a30305b29b0755b04b1ca591ba11e7d6121f3dfd0307ed5e730c0323b95f04663b234cdd18c776c578debbe390f22500061d78d808cdae51e89c0af10
+  data.tar.gz: a826644b2b8c4a1ceb3a12ed50ebae350bd6c0bc77e71a0a8571264c31f8903a1ee661a3366c0d1a7136680939f7ab70ded31a2be53124ca2589a7f11fc2f06f

data/lib/ffwd/plugin/google_cloud/hook.rb

@@ -15,6 +15,8 @@
 
 require 'json'
 
+require 'em/all'
+
 require_relative 'utils'
 
 require 'ffwd/flushing_output_hook'
@@ -42,9 +44,13 @@ module FFWD::Plugin::GoogleCloud
       # list of blocks waiting for a token.
       @pending = []
 
-      @api = "/cloudmonitoring/v2beta2/projects/#{@project_id}/timeseries:write"
+      @write_api = "/cloudmonitoring/v2beta2/projects/#{@project_id}/timeseries:write"
+      @md_api = "/cloudmonitoring/v2beta2/projects/#{@project_id}/metricDescriptors"
       @acquire = "/0.1/meta-data/service-accounts/default/acquire"
       @expire_threshold = 10
+
+      # cache of seen metric descriptors
+      @md_cache = {}
     end
 
     def with_token &block
@@ -97,9 +103,95 @@ module FFWD::Plugin::GoogleCloud
       not @api_c.nil? and not @metadata_c.nil?
     end
 
+    def verify_descriptors(metrics, &block)
+      missing = []
+
+      metrics[:timeseries].each do |ts|
+        desc = ts[:timeseriesDesc]
+        name = desc[:metric]
+        missing << desc unless @md_cache[name]
+      end
+
+      if missing.empty?
+        return block.call
+      end
+
+      all = EM::All.new
+
+      # Example: {:metric=>"custom.cloudmonitoring.googleapis.com/lol", :labels=>{}}
+      missing.each do |m|
+        name = m[:metric]
+
+        body = {
+         :name => m[:metric], :description => m[:metric], :labels => Utils.extract_labels(m[:labels]),
+         :typeDescriptor => {:metricType => "gauge", :valueType => "double"}
+        }
+
+        log.info "Creating descriptor for: #{name}"
+
+        all << with_token{|token|
+          head = Hash[HEADER_BASE]
+          head['Authorization'] = "Bearer #{token}"
+          p = new_proxy(api_c.post(:path => @md_api, :head => head, :body => JSON.dump(body)))
+
+          p.callback do
+            log.info "Created (caching): #{name}"
+            @md_cache[name] = true
+          end
+
+          p.errback do |error|
+            log.error "Failed to create descriptor (#{error}): #{body}"
+          end
+
+          p
+        }
+      end
+
+      # TODO: call block _after_ descriptors have been verified.
+      all.callback do
+        block.call
+      end
+
+      all.errback do |errors|
+        log.error "Failed to create descriptors: #{errors}"
+      end
+
+      SingleProxy.new all
+    end
+
+    # avoid pipelining requests by creating a new handle for each
+    def api_c
+      EM::HttpRequest.new(@api_url)
+    end
+
     def connect
       @metadata_c = EM::HttpRequest.new(@metadata_url)
       @api_c = EM::HttpRequest.new(@api_url)
+
+      with_token do |token|
+        log.info "Downloading metric descriptors"
+
+        head = Hash[HEADER_BASE]
+        head['Authorization'] = "Bearer #{token}"
+        req = @api_c.get(:path => @md_api, :head => head)
+        p = new_proxy(req)
+
+        p.callback do
+          res = JSON.load(req.response)
+
+          log.info "Downloaded #{res["metrics"].length} descriptor(s)"
+
+          res["metrics"].each do |d|
+            @md_cache[d["name"]] = true
+          end
+        end
+
+        p.errback do |error|
+          log.error "Failed to download metric descriptors: #{error}"
+        end
+
+        p
+      end
     end
 
     def close
@@ -111,21 +203,26 @@ module FFWD::Plugin::GoogleCloud
     end
 
     def send metrics
-      request = {
-        "timeseries" => Utils.make_timeseries(metrics)
-      }
+      dropped, timeseries = Utils.make_timeseries(metrics)
+
+      if dropped > 0
+        log.warning "Dropped #{dropped} points (duplicate entries)"
+      end
 
+      request = {:timeseries => timeseries}
       metrics = JSON.dump(request)
 
-      with_token do |token|
-        head = Hash[HEADER_BASE]
-        head['Authorization'] = "Bearer #{token}"
+      verify_descriptors(request) do
+        with_token do |token|
+          head = Hash[HEADER_BASE]
+          head['Authorization'] = "Bearer #{token}"
 
-        if log.debug?
-          log.debug "Sending: #{metrics}"
-        end
+          if log.debug?
+            log.debug "Sending: #{metrics}"
+          end
 
-        new_proxy @api_c.post(:path => @api, :head => head, :body => metrics)
+          new_proxy api_c.post(:path => @write_api, :head => head, :body => metrics)
+        end
       end
     end
 
@@ -157,6 +254,45 @@ module FFWD::Plugin::GoogleCloud
     end
   end
 
+  class SingleProxy
+    attr_reader :error
+
+    def initialize p
+      @callbacks = []
+      @errbacks = []
+
+      p.callback do
+        call
+      end
+
+      p.errback do |error|
+        err error
+      end
+    end
+
+    def callback &block
+      @callbacks << block
+    end
+
+    def errback &block
+      @errbacks << block
+    end
+
+    def call
+      @callbacks.each(&:call).clear
+    end
+
+    def err error
+      return if @errbacks.empty?
+
+      @error = error
+
+      @errbacks.each do |cb|
+        cb.call error
+      end.clear
+    end
+  end
+
   class CallbackProxy
     attr_reader :error
 
@@ -179,8 +315,12 @@ module FFWD::Plugin::GoogleCloud
 
     def err error
       return if @errbacks.empty?
+
       @error = error
-      @errbacks.each(&:call).clear
+
+      @errbacks.each do |cb|
+        cb.call error
+      end.clear
     end
 
     def into other

data/lib/ffwd/plugin/google_cloud/utils.rb

@@ -12,13 +12,32 @@
 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 # License for the specific language governing permissions and limitations under
 # the License.
+require 'set'
 
 module FFWD::Plugin::GoogleCloud
+  CUSTOM_PREFIX = "custom.cloudmonitoring.googleapis.com"
+
   module Utils
     def self.make_timeseries buffer
-      buffer.map do |m|
-        {:timeseriesDesc => make_desc(m), :point => make_point(m)}
+      # we are not allowed to send duplicate data.
+      seen = Set.new
+
+      result = []
+      dropped = 0
+
+      buffer.each do |m|
+        d = make_desc(m)
+
+        if seen.member?(d[:metric])
+          dropped += 1
+          next 
+        end
+
+        seen.add(d[:metric])
+        result << {:timeseriesDesc => make_desc(m), :point => make_point(m)}
       end
+
+      [dropped, result]
     end
 
     def self.make_point m
@@ -27,13 +46,31 @@ module FFWD::Plugin::GoogleCloud
     end
 
     def self.make_desc m
-      {:metric => m.key, :labels => make_labels(m)}
+      {:metric => make_key(m), :labels => make_labels(m)}
+    end
+
+    def self.make_key m
+      what ||= m.attributes[:what]
+      what ||= m.attributes["what"]
+
+      if what.nil?
+        "#{CUSTOM_PREFIX}/#{m.key}"
+      else
+        "#{CUSTOM_PREFIX}/#{m.key}.#{what}"
+      end
     end
 
     def self.make_labels m
-      labels = Hash[m.attributes]
-      labels["host"] = m.host
+      labels = Hash[m.attributes.select{|k, v| k.to_s != "what"}.map{|k, v|
+        ["#{CUSTOM_PREFIX}/#{k}", v]
+      }]
+
+      #labels["#{CUSTOM_PREFIX}/host"] = m.host
       labels
     end
+
+    def self.extract_labels source
+      source.map{|k, v| {:key => k, :description => k}}
+    end
   end
 end

data/lib/ffwd/plugin/google_cloud/version.rb

@@ -16,7 +16,7 @@
 module FFWD
   module Plugin
     module GoogleCloud
-      VERSION = "0.4.0"
+      VERSION = "0.4.1"
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ffwd-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.0
+  version: 0.4.1
 platform: ruby
 authors:
 - John-John Tedro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-10 00:00:00.000000000 Z
+date: 2015-03-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.1
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.0
+        version: 0.4.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
@@ -73,10 +73,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- lib/ffwd/plugin/google_cloud.rb
 - lib/ffwd/plugin/google_cloud/hook.rb
 - lib/ffwd/plugin/google_cloud/utils.rb
 - lib/ffwd/plugin/google_cloud/version.rb
-- lib/ffwd/plugin/google_cloud.rb
 homepage: https://github.com/spotify/ffwd
 licenses:
 - Apache 2.0
@@ -97,7 +97,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.14
+rubygems_version: 2.2.2
 signing_key: 
 specification_version: 4
 summary: Google Cloud Monitoring support for FFWD.