ffwd-google-cloud 0.4.2 → 0.4.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 85933ad63b1c5d571fc088d5d36083a703b132c5
-  data.tar.gz: 7efaf4b8b585240dca680c11a18fa12a291ad3a8
+  metadata.gz: 2460bbc3a9e1c03e415c76926df07e8fb7f1f03a
+  data.tar.gz: 4c8c48826dde0239a18b0f50aa15342420a4d5da
 SHA512:
-  metadata.gz: 2d7b1e2c7190804535447aa2b3a6e41d78b040f7ad9c87cca55b8ce4d46f373329945392bb92a627adde052ed1cb6a14a57479f88cd1f7bd846846df4d7ce97d
-  data.tar.gz: f54fc5d8ca85a7345d281a897b6d4a4b8a394451e34033eff4c5f1c42a3efcba4198a720bd6c5391f8ead7eef7d45197c4cea221190bfc92acceecf051e20e5b
+  metadata.gz: f209e4155c96938adc579c63eb0fd8ccdf05b9f10eab55139f4c541dfd47db23bdf7a5d8f5a8795a863932b1b2cd63c1911f747dc3b611e0415813dc0f7995d3
+  data.tar.gz: 899ee149a5e79621a90ca817b9d79446784f21f7c38bfcd197f2e25817ca05ce5e1f8097566d3fcdb05ea1d158de6e8c577057e2044116add5f4011a42b71754

data/lib/ffwd/plugin/google_cloud/hook.rb

@@ -36,9 +36,10 @@ module FFWD::Plugin::GoogleCloud
       @project = c[:project]
       @client_id = c[:client_id]
       @scope = c[:scope]
+      @debug = c[:debug]
 
-      @api_c = nil
-      @metadata_c = nil
+      @api_pipeline = nil
+      @metadata_pipeline = nil
       @token = nil
       @expires_at = nil
       # list of blocks waiting for a token.
@@ -71,7 +72,7 @@ module FFWD::Plugin::GoogleCloud
 
       log.debug "Requesting token"
 
-      http = @metadata_c.get(
+      http = @metadata_pipeline.get(
         :path => @acquire,
         :query => {:client_id => @client_id, :scope => @scope})
 
@@ -100,7 +101,7 @@ module FFWD::Plugin::GoogleCloud
     end
 
     def active?
-      not @api_c.nil? and not @metadata_c.nil?
+      not @api_pipeline.nil? and not @metadata_pipeline.nil?
     end
 
     def verify_descriptors(metrics, &block)
@@ -122,17 +123,21 @@ module FFWD::Plugin::GoogleCloud
       missing.each do |m|
         name = m[:metric]
 
-        body = {
-         :name => m[:metric], :description => m[:metric], :labels => Utils.extract_labels(m[:labels]),
+        descriptor = {
+         :name => m[:metric], :description => "",
+         :labels => Utils.extract_labels(m[:labels]),
          :typeDescriptor => {:metricType => "gauge", :valueType => "double"}
         }
 
+        descriptor = JSON.dump(descriptor)
+
         log.info "Creating descriptor for: #{name}"
 
         all << with_token{|token|
           head = Hash[HEADER_BASE]
           head['Authorization'] = "Bearer #{token}"
-          p = new_proxy(api_c.post(:path => @md_api, :head => head, :body => JSON.dump(body)))
+          p = new_proxy(api_request.post(
+            :path => @md_api, :head => head, :body => descriptor))
 
           p.callback do
             log.info "Created (caching): #{name}"
@@ -140,7 +145,7 @@ module FFWD::Plugin::GoogleCloud
           end
 
           p.errback do |error|
-            log.error "Failed to create descriptor (#{error}): #{body}"
+            log.error "Failed to create descriptor (#{error}): #{descriptor}"
           end
 
           p
@@ -159,57 +164,51 @@ module FFWD::Plugin::GoogleCloud
       SingleProxy.new all
     end
 
+    def metadata_request
+      if @debug
+        return DebugRequest.new(@metadata_url, :response => {
+          :accessToken => "FAKE", :expiresAt => (Time.now.to_i + 3600)})
+      end
+
+      EM::HttpRequest.new(@metadata_url)
+    end
+
     # avoid pipelining requests by creating a new handle for each
-    def api_c
+    def api_request
+      if @debug
+        return DebugRequest.new(@api_url, :responses => {
+          [:get, @md_api] => {"metrics" => []},
+          [:post, @write_api] => {}
+        })
+      end
+
       EM::HttpRequest.new(@api_url)
     end
 
     def connect
-      @metadata_c = EM::HttpRequest.new(@metadata_url)
-      @api_c = EM::HttpRequest.new(@api_url)
-
-      with_token do |token|
-        log.info "Downloading metric descriptors"
-
-        head = Hash[HEADER_BASE]
-        head['Authorization'] = "Bearer #{token}"
-        req = @api_c.get(:path => @md_api, :head => head)
-        p = new_proxy(req)
-
-        p.callback do
-          res = JSON.load(req.response)
-
-          log.info "Downloaded #{res["metrics"].length} descriptor(s)"
-
-          res["metrics"].each do |d|
-            @md_cache[d["name"]] = true
-          end
-        end
-
-        p.errback do |error|
-          log.error "Failed to download metric descriptors: #{error}"
-        end
-
-        p
-      end
+      @metadata_pipeline = metadata_request
+      @api_pipeline = api_request
+      populate_metadata_cache
     end
 
     def close
-      @metadata_c.close if @metadata_c
-      @api_c.close if @api_c
+      @metadata_pipeline.close if @metadata_pipeline
+      @api_pipeline.close if @api_pipeline
 
-      @metadata_c = nil
-      @api_c = nil
+      @metadata_pipeline = nil
+      @api_pipeline = nil
     end
 
     def send metrics
       dropped, timeseries = Utils.make_timeseries(metrics)
 
+      common_labels = Utils.make_common_labels(metrics)
+
       if dropped > 0
         log.warning "Dropped #{dropped} points (duplicate entries)"
       end
 
-      request = {:timeseries => timeseries}
+      request = {:commonLabels => common_labels, :timeseries => timeseries}
       metrics = JSON.dump(request)
 
       verify_descriptors(request) do
@@ -221,7 +220,8 @@ module FFWD::Plugin::GoogleCloud
             log.debug "Sending: #{metrics}"
           end
 
-          new_proxy api_c.post(:path => @write_api, :head => head, :body => metrics)
+          new_proxy api_request.post(
+            :path => @write_api, :head => head, :body => metrics)
         end
       end
     end
@@ -252,6 +252,35 @@ module FFWD::Plugin::GoogleCloud
     def reporter_meta
       {:component => :google_cloud}
     end
+
+    private
+
+    def populate_metadata_cache
+      with_token do |token|
+        log.info "Downloading metric descriptors"
+
+        head = Hash[HEADER_BASE]
+        head['Authorization'] = "Bearer #{token}"
+        req = @api_pipeline.get(:path => @md_api, :head => head)
+        p = new_proxy(req)
+
+        p.callback do
+          res = JSON.load(req.response)
+
+          log.info "Downloaded #{res["metrics"].length} descriptor(s)"
+
+          res["metrics"].each do |d|
+            @md_cache[d["name"]] = true
+          end
+        end
+
+        p.errback do |error|
+          log.error "Failed to download metric descriptors: #{error}"
+        end
+
+        p
+      end
+    end
   end
 
   class SingleProxy
@@ -299,18 +328,30 @@ module FFWD::Plugin::GoogleCloud
     def initialize
       @callbacks = []
       @errbacks = []
+      @called = false
     end
 
     def callback &block
+      if @called
+        block.call if @called == :call
+        return
+      end
+
       @callbacks << block
     end
 
     def errback &block
+      if @called
+        block.call if @called == :err
+        return
+      end
+
       @errbacks << block
     end
 
     def call
       @callbacks.each(&:call).clear
+      @called = :call
     end
 
     def err error
@@ -321,6 +362,8 @@ module FFWD::Plugin::GoogleCloud
       @errbacks.each do |cb|
         cb.call error
       end.clear
+
+      @called = :err
     end
 
     def into other
@@ -328,4 +371,61 @@ module FFWD::Plugin::GoogleCloud
       callback { other.call }
     end
   end
+
+  class DebugRequest
+    include FFWD::Logging
+
+    class Callback
+      class ResponseHeader
+        attr_reader :status
+
+        def initialize status
+          @status = status
+        end
+      end
+
+      attr_reader :response
+
+      def initialize response
+        @response = response
+      end
+
+      def callback &block
+        block.call
+      end
+
+      def response_header
+        ResponseHeader.new 200
+      end
+
+      def errback &block; end
+    end
+
+    def initialize url, params={}
+      @url = url
+      @response = JSON.dump(params[:response] || {})
+      @responses = Hash[(params[:responses] || {}).map{|k, v| [k, JSON.dump(v)]}]
+    end
+
+    def post params={}
+      log_request :post, params
+    end
+
+    def get params={}
+      log_request :get, params
+    end
+
+    # do nothing
+    def close; end
+
+    private
+
+    def log_request method, params
+      path = params.delete(:path) || ""
+      body = params.delete(:body)
+      log.debug "#{method} #{@url}#{path}: #{params}"
+      puts JSON.pretty_generate(JSON.load(body || "{}")) if body
+      Callback.new(@responses[[method, path]] || @response)
+    end
+  end
 end

data/lib/ffwd/plugin/google_cloud/utils.rb

@@ -18,6 +18,11 @@ module FFWD::Plugin::GoogleCloud
   CUSTOM_PREFIX = "custom.cloudmonitoring.googleapis.com"
 
   module Utils
+    def self.make_common_labels buffer
+      return nil if buffer.empty?
+      make_labels buffer.first.fixed_attr
+    end
+
     def self.make_timeseries buffer
       # we are not allowed to send duplicate data.
       seen = Set.new
@@ -46,11 +51,11 @@ module FFWD::Plugin::GoogleCloud
     end
 
     def self.make_desc m
-      {:metric => make_key(m), :labels => make_labels(m)}
+      {:metric => make_key(m), :labels => make_labels(m.external_attr)}
     end
 
     def self.make_key m
-      attributes = Hash[m.attributes]
+      attributes = Hash[m.external_attr]
 
       entries = []
 
@@ -68,17 +73,14 @@ module FFWD::Plugin::GoogleCloud
       end
     end
 
-    def self.make_labels m
-      labels = Hash[m.attributes.select{|k, v| k.to_s != "what"}.map{|k, v|
+    def self.make_labels attr
+      Hash[attr.select{|k, v| k.to_s != "what"}.map{|k, v|
         ["#{CUSTOM_PREFIX}/#{k}", v]
       }]
-
-      #labels["#{CUSTOM_PREFIX}/host"] = m.host
-      labels
     end
 
     def self.extract_labels source
-      source.map{|k, v| {:key => k, :description => k}}
+      source.map{|k, v| {:key => k, :description => ""}}
     end
   end
 end

data/lib/ffwd/plugin/google_cloud/version.rb

@@ -16,7 +16,7 @@
 module FFWD
   module Plugin
     module GoogleCloud
-      VERSION = "0.4.2"
+      VERSION = "0.4.3"
     end
   end
 end

data/lib/ffwd/plugin/google_cloud.rb

@@ -34,6 +34,7 @@ module FFWD::Plugin::GoogleCloud
   DEFAULT_SCOPE = "https://www.googleapis.com/auth/monitoring"
   DEFAULT_FLUSH_INTERVAL = 10
   DEFAULT_BUFFER_LIMIT = 100
+  DEFAULT_DEBUG = false
 
   def self.setup_output config
     if not config[:project_id]
@@ -53,6 +54,8 @@ module FFWD::Plugin::GoogleCloud
     config[:api_url] ||= DEFAULT_API_URL
     config[:flush_interval] ||= DEFAULT_FLUSH_INTERVAL
     config[:buffer_limit] ||= DEFAULT_BUFFER_LIMIT
+    # Fake all API interaction (output with log.debug)
+    config[:debug] ||= DEFAULT_DEBUG
 
     hook = Hook.new(config)
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ffwd-google-cloud
 version: !ruby/object:Gem::Version
-  version: 0.4.2
+  version: 0.4.3
 platform: ruby
 authors:
 - John-John Tedro
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-11 00:00:00.000000000 Z
+date: 2015-03-12 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: em-http-request
@@ -30,14 +30,14 @@ dependencies:
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.3
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - '='
       - !ruby/object:Gem::Version
-        version: 0.4.2
+        version: 0.4.3
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement