pencil 0.2.2 → 0.2.6

data/bin/pencil

@@ -1,6 +1,9 @@
 #!/usr/bin/env ruby
 $: << File.dirname($0) + "/../lib"
 
+require File.join(File.dirname(__FILE__), "..", "VERSION")
 require "dash"
 
+STDOUT.puts "pencil v#{PENCIL_VERSION} on port #{Dash::App.port}, PID #{$$}"
+
 Rack::Handler::Mongrel.run(Dash::App, :Port => Dash::App.port)

data/docs/pencil_options.md

@@ -77,6 +77,23 @@ These are options that go under the :config key in pencil configuration files.
   How many seconds before Time.now an end time is considered to still be 'now',
   for the purposes of adding meta-refresh and displaying time intervals.
 
+* :use_color: [Boolean, optional, default false]
+
+  In graphite 0.9.8 and earlier coloring of targets sucks. Coloring is not done
+  on a per-target basis, but rather by an additional parameter
+  "colorList". Pencil handles colors by appending a target's color parameter
+  (or a suitable default) onto this list. Targets are processed in order and
+  consume the first color available in colorList.
+
+  This sucks because nonexistent targets occasionally produced by pencil don't
+  consume their associated color, which screws up the coloring of every
+  subsequent metric.
+
+  The latest Graphite trunk supports color as a property of a target, set with
+  color(target, "COLOR"). If you are using a new graphite and want to take
+  advantage of this more accurate coloring technique set :use_color to true,
+  and bask in the glory of color correctness.
+
 ## <a name="gopts"/> Graph-level Options
 This is a list of the supported graph-level options for pencil, which
 correspond to request(image)-level options for graphite. These options are
@@ -103,11 +120,25 @@ minor annotations:
 
 ## Target-level Options
 This is a list of the supported target-level options for pencil. These are
-mosly a list of transformations graphite supports, including summation and
+mostly a list of transformations graphite supports, including summation and
 scaling of metrics. You can apply them to individual metrics, or lists of
 metrics. See the example configs for how this works. Also see the graphite
 composer for the effects of these options, many of which are untested.
 
+A note on the divideSeries case: for aggregate graphs, pencil takes the ratio
+of the sums, as opposed to the sums of the ratios, for ease of implementation
+and because it makes some sense. If you're not a fan of 80k URLs then you will
+agree. divideSeries should only ever be used with two targets, like this:
+
+    ? - stats.timers.mysql.innodb.threads_active_read.mean:
+      - stats.timers.mysql.innodb.threads_total_read.mean:
+    :
+      !omap
+      - :divideSeries:
+
+(Notice the omap, as opposed to a hash, to impose an order in which options are
+applied)
+
 ### Combinations
 These functions take an arbitrary number of targets (usually simple metrics)
 for arguments.
@@ -139,7 +170,7 @@ for arguments.
 * stdev
 * asPercent
 * diffSeries
-* ratio
+* divideSeries (NOTE: takes exactly 1 series as divisor)
 
 ### Filters
 Most of these options take a single argument.
@@ -172,5 +203,7 @@ Most of these options take a single argument.
 * threshold
 * color
 
-Note: key and color are interpreted differently from the other options, which
-are more simply translated.
+Note: key is interpreted differently from the other options, which are more
+simply translated.
+
+color's interpretation depends on whether :use_color is set.

data/examples/dashboards.yml

@@ -4,12 +4,30 @@
     title: syncstorage_overview
     graphs:
       - syncstorage_qps:
-        # hosts: ["sync*_web"]
       - syncstorage_401:
       - syncstorage_exceptions:
       - cpu_usage:
+      - memory_usage:
       - network_traffic:
     hosts: ["sync*_web", "wp-web*"]
+  syncreg:
+    title: syncreg_overview
+    graphs:
+      - syncreg_qps:
+      - syncreg_exceptions:
+      - cpu_usage:
+      - memory_usage:
+      - network_traffic:
+    hosts: ["wp-reg*"]
+  syncsreg:
+    title: syncsreg_overview
+    graphs:
+      - syncsreg_qps:
+      - syncsreg_exceptions:
+      - cpu_usage:
+      - memory_usage:
+      - network_traffic:
+    hosts: ["wp-sreg*"]
   ldap_slaves:
     title: LDAP slaves
     graphs:
@@ -34,3 +52,9 @@
       - cpu_usage:
       - network_traffic:
     hosts: ["sync*_db", "wp-db*"]
+  sync_node_alloc:
+    title: sync node allocation
+    graphs:
+      - sync_node_alloc:
+      - sync_node_actives:
+    hosts: ["wp-adm01"]

data/examples/graphs.yml

@@ -15,12 +15,6 @@
       syncstorage.request_rate.404:
         :key: 404
         :color: purple
-      syncstorage.request_rate.500:
-        :key: 500
-        :color: yellow
-      syncstorage.request_rate.502:
-        :key: 502
-        :color: "#cc0000"
       syncstorage.request_rate.503:
         :key: 503
         :color: red
@@ -31,7 +25,7 @@
     title: "syncstorage 401s"
     targets:
       syncstorage.request_rate.401:
-        :key: 200
+        :key: 401
         :color: brown
     hosts: ["sync*_web", "test*_web", "wp-web*"]
 
@@ -43,6 +37,64 @@
         :color: red
     hosts: ["sync*_web", "test*_web", "wp-web*"]
 
+  syncreg_qps:
+    title: "syncreg qps"
+    targets:
+      syncreg.request_rate.200:
+        :key: 200
+        :color: green
+      syncreg.request_rate.302:
+        :key: 302
+        :color: blue
+      syncreg.request_rate.401:
+        :key: 401
+        :color: brown
+      syncreg.request_rate.404:
+        :key: 404
+        :color: purple
+      syncreg.request_rate.503:
+        :key: 503
+        :color: red
+    areaMode: stacked
+    hosts: ["wp-reg*"]
+
+  syncreg_exceptions:
+    title: "syncreg exceptions"
+    targets:
+      syncreg.log.exception:
+        :key: exceptions/s
+        :color: red
+    hosts: ["wp-reg*"]
+
+  syncsreg_qps:
+    title: "syncsreg qps"
+    targets:
+      syncsreg.request_rate.200:
+        :key: 200
+        :color: green
+      syncsreg.request_rate.302:
+        :key: 302
+        :color: blue
+      syncsreg.request_rate.401:
+        :key: 401
+        :color: brown
+      syncsreg.request_rate.404:
+        :key: 404
+        :color: purple
+      syncsreg.request_rate.503:
+        :key: 503
+        :color: red
+    areaMode: stacked
+    hosts: ["wp-sreg*"]
+
+  syncsreg_exceptions:
+    title: "syncsreg exceptions"
+    targets:
+      syncsreg.log.exception:
+        :key: exceptions/s
+        :color: red
+    hosts: ["wp-sreg*"]
+
   cpu_usage:
     title: "cpu usage"
     targets:
@@ -145,3 +197,27 @@
     bgcolor: "#AFFFFF"
     vtitle: tvtnolgrf
     lineWidth: 20
+
+  sync_node_alloc:
+    title: "node alloc rate"
+    targets:
+      sync.users.alloc_rate.scl2:
+        :key: scl2 alloc/s
+        :color: green
+      sync.users.alloc_rate.phx1:
+        :key: phx1 alloc/s
+        :color: blue
+ 
+    hosts: ["wp-adm01"]
+
+  sync_node_actives:
+    title: "node actives"
+    targets:
+      sync.users.active.scl2:
+        :key: scl2 actives
+        :color: green
+      sync.users.active.phx1:
+        :key: phx1 actives
+        :color: blue
+ 
+    hosts: ["wp-adm01"]

data/examples/pencil.yml

@@ -19,3 +19,6 @@
   :date_format: "%X %x" # strftime
   :metric_format: "%m.%c.%h" #%m metric, %c cluster, %h host
   :now_threshold: 300 # Time.at(Time.now - now_threshold) considered Time.now
+
+  # set this if you have a new graphite that supports color(series, "color")
+  # :use_color: true

data/lib/dash.rb

@@ -15,6 +15,7 @@ require "yaml"
 require "chronic"
 require "chronic_duration"
 require "optparse"
+require "rubyfixes"
 
 # fixme style.css isn't actually cached, you need to set something up with
 # rack to cache static files
@@ -32,6 +33,7 @@ module Dash
     use Rack::Session::Cookie, :expire_after => 126227700 # 4 years
     set :root, File.dirname(__FILE__)
     set :static, true
+    set :logging, true
 
     def initialize(settings={})
       super
@@ -73,7 +75,11 @@ module Dash
 
     get %r[^/(dash/?)?$] do
       @no_graphs = true
-      redirect '/dash/global'
+      if settings.config.clusters.size == 1
+        redirect "/dash/#{settings.config.clusters.first}"
+      else
+        redirect '/dash/global'
+      end
     end
 
     get '/dash/:cluster/:dashboard/:zoom/?' do

data/lib/helpers.rb

@@ -188,7 +188,7 @@ module Dash::Helpers
 
   def permalink
     return "" unless @stime && @duration
-    format = "%x %X" # chronic understands this
+    format = "%F %T" # chronic REALLY understands this
     url = request.path + "?"
     url << "&start=#{@stime.strftime(format)}"
     url << "&duration=#{ChronicDuration.output(@duration)}"

data/lib/models/dashboard.rb

@@ -15,7 +15,7 @@ module Dash::Models
       @graph_opts = {}
       params["graphs"].each do |name|
         # graphs map to option hashes
-        if name.instance_of?(Hash)
+        if name.instance_of?(Hash) # could be YAML::Omap
           g = Graph.find(name.keys.first) # should only be one key
           @graph_opts[g] = name[name.keys.first]||{}
         else

data/lib/models/graph.rb

@@ -51,7 +51,7 @@ module Dash::Models
         # calculate
         "movingAverage", "stdev", "asPercent"
         x.call
-      when "diffSeries", "ratio"
+      when "diffSeries", "divideSeries"
         z.call
         # filters
       when "mostDeviant"
@@ -75,7 +75,7 @@ module Dash::Models
       when "substr", "threshold"
         x.call
       when "color"
-        str #color is handled elsewhere
+        @params[:use_color] ? "color(#{str}, \"#{arg}\")" : str
       else
         raise "BAD FUNC #{func}" unless pass
         str
@@ -83,7 +83,6 @@ module Dash::Models
     end
 
     # inner means we're dealing with a complex key; @params will be applied
-    # later on
     def handle_metric(name, opts, inner=false)
       ret = name.dup
       if inner
@@ -136,18 +135,24 @@ module Dash::Models
 
       target = []
       colors = []
-      #fixme code duplication
+      #FIXME code duplication
       if opts[:sum] == :global
         @params["targets"].each do |stat_name, opts|
-          z = opts.dup
-          # opts['key'] ||= stat_name
+          z = Marshal.load(Marshal.dump(opts))
           z[:key] ||= stat_name
           #######################
           if stat_name.instance_of?(Array)
             metric = stat_name.map do |m|
-              mm = compose_metric(m.keys.first, clusters.to_a.join(","),
-                           hosts.to_a.join(","))
-              handle_metric(mm, m[m.keys.first], true)
+              mm = compose_metric(m.keys.first,
+                           "{#{clusters.to_a.join(',')}}",
+                           "{#{hosts.to_a.join(',')}}")
+
+              if z.keys.member?(:divideSeries)
+                handle_metric(translate(:sumSeries, mm),
+                       m[m.keys.first], true)
+              else
+                handle_metric(mm, m[m.keys.first], true)
+              end
             end.join(",")
           else
             metric = "#{stat_name}.{#{clusters.to_a.join(',')}}" +
@@ -156,30 +161,68 @@ module Dash::Models
           end
           #######################
           z[:key] = "global #{z[:key]}"
-          target << handle_metric("sumSeries(#{metric})", z)
-          colors << next_color(colors, z[:color])
+          # target << handle_metric(translate(:sumSeries, metric), z)
+
+          if z.keys.member?(:divideSeries) # special case
+              # apply divideSeries, sumSeries then other options
+              res = translate(:divideSeries, metric)
+              res = translate(:sumSeries, res)
+              z.delete(:divideSeries)
+              h = YAML::Omap.new
+              z.each { |k,v| h[k] = v unless k == :divideSeries }
+              target << handle_metric(res, h)
+            else
+              target << handle_metric(translate(:sumSeries, metric), z)
+            end
+          if !@params[:use_color] ||
+            (!z[:color] && @params[:use_color])
+            colors << next_color(colors, z[:color])
+          end
         end # @params["targets"].each
       elsif opts[:sum] == :cluster # one line per cluster/metric
         clusters.each do |cluster|
           @params["targets"].each do |stat_name, opts|
-            z = opts.dup
+            z = Marshal.load(Marshal.dump(opts))
             metrics = []
-            hosts.each do |host|
-              #######################
-              if stat_name.instance_of?(Array)
-                metrics << stat_name.map do |m|
-                  mm = compose_metric(m.keys.first, cluster, host)
+            #######################
+            h = "{#{hosts.to_a.join(',')}}"
+            if stat_name.instance_of?(Array)
+              metrics << stat_name.map do |m|
+                mm = compose_metric(m.keys.first, cluster, h)
+                # note: we take the ratio of the sums in this case, instead of
+                # the sums of the ratios
+                if z.keys.member?(:divideSeries)
+                  # divideSeries is picky about the number of series given as
+                  # arguments, so sum them in this case
+                  handle_metric(translate(:sumSeries, mm),
+                         m[m.keys.first], true)
+                else
                   handle_metric(mm, m[m.keys.first], true)
-                end.join(",")
-              else
-                metrics << handle_metric(compose_metric(stat_name, cluster, host), {}, true)
-              end
-              #######################
-            end # hosts.each
-
+                end
+              end.join(",")
+            else
+              metrics << handle_metric(compose_metric(stat_name,
+                                               cluster, h), {}, true)
+            end
+            #######################
             z[:key] = "#{cluster} #{z[:key]}"
-            target << handle_metric("sumSeries(#{metrics.join(',')})", z)
-            colors << next_color(colors, z[:color])
+
+            if z.keys.member?(:divideSeries) # special case
+              # apply divideSeries, sumSeries then other options
+              res = translate(:divideSeries, metrics.join(','))
+              res = translate(:sumSeries, res)
+              z.delete(:divideSeries)
+              h = YAML::Omap.new
+              z.each { |k,v| h[k] = v unless k == :divideSeries }
+              target << handle_metric(res, h)
+            else
+              target << handle_metric(translate(:sumSeries,
+                                      metrics.join(',')), z)
+            end
+
+            if !@params[:use_color] || (!z[:color] && @params[:use_color])
+              colors << next_color(colors, z[:color])
+            end
           end # metrics.each
         end # clusters.each
       else # one line per {metric,host,colo}
@@ -199,9 +242,11 @@ module Dash::Models
               #################
 
               if label =~ /\*/
-                # for this particular type of graph, don't display a legend
+                # for this particular type of graph, don't display a legend,
+                # and color with abandon
                 url_opts[:hideLegend] = true
-                z = opts.dup
+                z = Marshal.load(Marshal.dump(opts))
+                z.delete(:color)
                 # fixme proper labeling... maybe
                 # With wildcards let graphite construct the legend (or not).
                 # Since we're handling wildcards we don't know how many
@@ -211,10 +256,13 @@ module Dash::Models
                 target << handle_metric(metric, z)
                 colors.concat(@params[:default_colors]) if colors.empty?
               else
-                z = opts.dup
-                z[:key] = "#{host}/#{cluster} #{opts[:key]}"
+                z = Marshal.load(Marshal.dump(opts))
+                z[:key] = "#{host}/#{cluster} #{z[:key]}"
                 target << handle_metric(metric, z)
-                colors << next_color(colors, opts[:color])
+                if !@params[:use_color] ||
+                  (!opts[:color] && @params[:use_color])
+                  colors << next_color(colors, opts[:color])
+                end
               end
             end
           end
@@ -259,13 +307,27 @@ module Dash::Models
       return metrics.flatten.map { |x| x.split(".") }
     end
 
+    # FIXMEEEE needs to be fixed for different metric formats
     def hosts_clusters
       metrics = expand
       clusters = Set.new
 
-      # field -1 is the host name, and -2 is its cluster
-      hosts = metrics.map do |x|
-        Host.new(x[-1], @params.merge({ "cluster" => x[-2] }))
+      f = @params[:metric_format].dup.split("%m")
+      first = f.first.split(".")
+      last = f.last.split(".")
+      ci = hi = nil
+      first.each_with_index do |v, i|
+        ci = i if v.match("%c")
+        hi = i if v.match("%h")
+      end
+      unless ci && hi
+        last.reverse.each_with_index do |v, i|
+          ci = -1 -i if v.match("%c")
+          hi = -1 -i if v.match("%h")
+        end
+      end
+      hosts = metrics.map do |m|
+        Host.new(m[hi], @params.merge({ "cluster" => m[ci] }))
       end.uniq
 
       # filter by what matches the graph definition
@@ -287,7 +349,7 @@ module Dash::Models
         default_colors << preferred_color
       end
 
-      weights = Hash.new { |h, k| h[k] = 0 }
+      weights = Hash.new(0)
       colors.each do |c|
         weights[c] += 1
       end

data/lib/rubyfixes.rb

@@ -0,0 +1,8 @@
+class YAML::Omap
+  def keys
+    self.map {|k,v| k}
+  end
+  def values
+    self.map {|k,v| v}
+  end
+end

data/lib/views/cluster.erb

@@ -16,8 +16,11 @@
     end %>
 <br>
 <% end %>
+
+<% if (hosts.to_set - seen_hosts).size > 0 %>
 <h3>Other Hosts (not associated with a dashboard)</h3>
 <%= (hosts.to_set - seen_hosts).sort.collect do |h|
     href = append_query_string("/host/#{@cluster}/#{h}")
     "<li><a href=\"#{href}\">#{h}</a></li>"
     end %>
+<% end %>

data/lib/views/dash-cluster-zoom.erb

@@ -15,14 +15,6 @@
   </div>
 </div>
 
-<div class="graphsection" style="width:<%= @zoom.width(merge_opts) %>;">
-  <a name="by_host">
-    <span class="graphtitle"><%= @zoom.name %> / <%= @cluster %> :: by host</span>
-    <div class="graph">
-      <img src="<%= @dash.render_cluster_graph(@zoom, @cluster, :zoom => true, :dynamic_url_opts => merge_opts) %>">
-    </div>
-</div>
-
 <% hosts.each do |host| %>
 <div class="graphsection" style="width:<%= @zoom.width(merge_opts) %>;">
   <a name="<%= host %>">

data/lib/views/layout.erb

@@ -3,7 +3,7 @@
     <%= css_url %>
     <link href="/favicon.ico" rel="icon" type="text/x-icon">
     <link href="/favicon.ico" rel="shortcut icon" type="text/x-icon">
-    <%= refresh %>
+    <%= refresh unless @no_graphs %>
   </head>
   <body bgcolor="black">
 
@@ -51,5 +51,7 @@
     </div>
 
   </body>
-  <div id="footer" style="color:green;">generated <%= @request_time %></div>
+  <div id="footer" style="color:green;">
+    generated <%= @request_time %> by <a href="https://github.com/petef/pencil">pencil</a> v<%= PENCIL_VERSION %>
+  </div>
 </html>

data/lib/views/partials/cluster_switcher.erb

@@ -7,6 +7,7 @@ clusters:
 <%= if @cluster == "global"
     " (global)"
     else
+    # this sub works because the cluster should be first
     " <a href=\"#{append_query_string(request.path.sub(@cluster, 'global'))}\">(global)</a>"
     end %>
 <br>

data/lib/views/partials/dash_switcher.erb

@@ -1,6 +1,7 @@
 <span class="graphtitle">
   dashboards:
   <%= @dashboards.sort.collect do |d|
+      # this works unless you have a cluster with the same name as a dashboard
       sub = append_query_string(request.path.sub(@dash.name, d.name))
       @dash == d ? "#{d}" : "<a href=\"#{sub}\">#{d}</a>"
       end.join(" ") %>

data/lib/views/partials/graph_switcher.erb

@@ -1,7 +1,10 @@
 <span class="graphtitle">
   graphs:
   <%= @dash.graphs.map { |g| g.name }.collect do |g|
-     sub = append_query_string(request.path.sub(@zoom.name, g))
+     reversed = request.path.split("/").reverse
+     i = reversed.index(@zoom.name)
+     reversed[i] = g
+     sub = append_query_string(reversed.reverse.join('/'))
      @zoom.name == g ? "#{g}" : "<a href=\"#{sub}\">#{g}</a>"
      end.join(" ") %>
 </span>

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: pencil
 version: !ruby/object:Gem::Version 
-  hash: 19
+  hash: 27
   prerelease: false
   segments: 
   - 0
   - 2
-  - 2
-  version: 0.2.2
+  - 6
+  version: 0.2.6
 platform: ruby
 authors: 
 - Pete Fritchman
@@ -16,7 +16,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-07-12 00:00:00 -07:00
+date: 2011-08-02 00:00:00 -07:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -114,33 +114,34 @@ extensions: []
 extra_rdoc_files: []
 
 files: 
+- lib/dash.rb
 - lib/models.rb
-- lib/public/style.css
-- lib/public/favicon.ico
+- lib/helpers.rb
+- lib/namespace.rb
 - lib/config.rb
-- lib/models/host.rb
-- lib/models/graph.rb
-- lib/models/dashboard.rb
-- lib/models/base.rb
+- lib/views/dash-cluster.erb
 - lib/views/cluster.erb
-- lib/views/global.erb
-- lib/views/dash-cluster-zoom.erb
-- lib/views/dash-global-zoom.erb
-- lib/views/dash-global.erb
+- lib/views/layout.erb
 - lib/views/host.erb
-- lib/views/partials/graph_switcher.erb
-- lib/views/partials/hosts_selector.erb
+- lib/views/dash-global.erb
+- lib/views/dash-global-zoom.erb
+- lib/views/global.erb
 - lib/views/partials/input_boxes.erb
+- lib/views/partials/graph_switcher.erb
 - lib/views/partials/cluster_selector.erb
-- lib/views/partials/cluster_switcher.erb
 - lib/views/partials/dash_switcher.erb
-- lib/views/partials/cookies_form.erb
+- lib/views/partials/hosts_selector.erb
 - lib/views/partials/refresh_button.erb
-- lib/views/layout.erb
-- lib/views/dash-cluster.erb
-- lib/namespace.rb
-- lib/helpers.rb
-- lib/dash.rb
+- lib/views/partials/cookies_form.erb
+- lib/views/partials/cluster_switcher.erb
+- lib/views/dash-cluster-zoom.erb
+- lib/rubyfixes.rb
+- lib/public/favicon.ico
+- lib/public/style.css
+- lib/models/graph.rb
+- lib/models/dashboard.rb
+- lib/models/base.rb
+- lib/models/host.rb
 - lib/config.ru
 - docs/pencil_options.md
 - examples/pencil.yml