RubyGems - mouth - Versions diffs - 0.8.0 - Mend

mouth 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (41) hide show

data/.gitignore +7 -0
data/Capfile +26 -0
data/Gemfile +3 -0
data/MIT-LICENSE +20 -0
data/README.md +138 -0
data/Rakefile +19 -0
data/TODO +32 -0
data/bin/mouth +77 -0
data/bin/mouth-console +18 -0
data/bin/mouth-endoscope +18 -0
data/lib/mouth.rb +61 -0
data/lib/mouth/dashboard.rb +25 -0
data/lib/mouth/endoscope.rb +120 -0
data/lib/mouth/endoscope/public/222222_256x240_icons_icons.png +0 -0
data/lib/mouth/endoscope/public/application.css +464 -0
data/lib/mouth/endoscope/public/application.js +938 -0
data/lib/mouth/endoscope/public/backbone.js +1158 -0
data/lib/mouth/endoscope/public/d3.js +4707 -0
data/lib/mouth/endoscope/public/d3.time.js +687 -0
data/lib/mouth/endoscope/public/jquery-ui-1.8.16.custom.min.js +177 -0
data/lib/mouth/endoscope/public/jquery.js +4 -0
data/lib/mouth/endoscope/public/json2.js +480 -0
data/lib/mouth/endoscope/public/keymaster.js +163 -0
data/lib/mouth/endoscope/public/linen.js +46 -0
data/lib/mouth/endoscope/public/seven.css +68 -0
data/lib/mouth/endoscope/public/seven.js +291 -0
data/lib/mouth/endoscope/public/underscore.js +931 -0
data/lib/mouth/endoscope/views/dashboard.erb +67 -0
data/lib/mouth/graph.rb +58 -0
data/lib/mouth/instrument.rb +56 -0
data/lib/mouth/record.rb +72 -0
data/lib/mouth/runner.rb +89 -0
data/lib/mouth/sequence.rb +284 -0
data/lib/mouth/source.rb +76 -0
data/lib/mouth/sucker.rb +235 -0
data/lib/mouth/version.rb +3 -0
data/mouth.gemspec +28 -0
data/test/sequence_test.rb +163 -0
data/test/sucker_test.rb +55 -0
data/test/test_helper.rb +5 -0
metadata +167 -0

data/lib/mouth/endoscope/views/dashboard.erb ADDED Viewed

@@ -0,0 +1,67 @@
+<!DOCTYPE html>
+<html>
+<head>
+  <title>Mouth Endoscope</title>
+	<link rel="stylesheet" href="/application.css" type="text/css" media="all">
+	<link rel="stylesheet" href="/seven.css" type="text/css" media="all">
+	<script src="/jquery.js" type="text/javascript"></script>
+	<script src="/jquery-ui-1.8.16.custom.min.js" type="text/javascript"></script>
+	<script src="/json2.js" type="text/javascript"></script>
+	<script src="/underscore.js" type="text/javascript"></script>
+	<script src="/backbone.js" type="text/javascript"></script>
+	<script src="/d3.js" type="text/javascript"></script>
+	<script src="/d3.time.js" type="text/javascript"></script>
+	<script src="/seven.js" type="text/javascript"></script>
+	<script src="/keymaster.js" type="text/javascript"></script>
+	<script src="/application.js" type="text/javascript"></script>
+	<script src="/linen.js" type="text/javascript"></script>
+</head>
+<body>
+<section id="mouth-canvas">
+  <header id="mouth-header">
+    <h1>Mouth</h1>
+    <nav id="dashboards" data-dashboard>
+      <ul></ul>
+      <a href="#" class="add-dashboard">+</a>
+    </nav>
+  </header>
+  <section id="info-panel">
+    <div id="date-range-picker" class="current">
+      <h2>Time Span</h2>
+      <ul>
+        <li><label><input type="radio" name="time_span" value="2_hours" checked> 2 Hours</label></li>
+        <li><label><input type="radio" name="time_span" value="24_hours"> 24 Hours</label></li>
+        <li><label><input type="radio" name="time_span" value="7_days"> 7 Days</label></li>
+      </ul>
+      <h2>Starting At</h2>
+      <div class="starting-at">
+        <div class="if-current">
+          <a href="#" class="custom-date">2 hours ago</a>
+        </div>
+        <div class="unless-current">
+          <input type="text" name="starting_at" class="date-starting-at">
+          <div class="example">(Format: 2012-04-26 11:22)</div>
+          <div><a href="#" class="date-reset">Reset</a></div>
+        </div>
+      </div>
+      <h2>Ending At</h2>
+      <div class="ending-at">
+        <span>Now</span>
+      </div>
+    </div>
+  </section>
+  <section id="current-dashboard">
+    <ul id="grid"></ul>
+    <a href="#" class="add-graph">Add Graph</a>
+  </section>
+</section>
+</body>
+</html>

data/lib/mouth/graph.rb ADDED Viewed

@@ -0,0 +1,58 @@
+module Mouth
+   # {
+   #   :dashboard_id => BSON::ObjectId,
+   #   :position => {
+   #     :top => 0,
+   #     :left => 0,
+   #     :height => 7,
+   #     :width => 20
+   #   },
+   #   :kind => 'counters' || 'timer',
+   #   :sources => ["auth.authentications"]
+   # }
+  class Graph < Record
+    #
+    def save
+      bson_object_id_ize(:dashboard_id) do
+        super
+      end
+    end
+    # Options:
+    #  - :start_time (Time object)
+    #  - :end_time (Time object)
+    #  - :granularity_in_minutes
+    def data(opts = {})
+      sources = self.attributes[:sources] || []
+      seq_opts = {:kind => self.attributes[:kind].to_sym}.merge(opts)
+      sequence = Sequence.new(sources, seq_opts)
+      seqs = sequence.sequences
+      seqs.map do |seq|
+        {
+          :data => seq[1],
+          :start_time => sequence.start_time_epoch,
+          :source => seq[0]
+        }
+      end
+    end
+    #
+    def bson_object_id_ize(*args)
+      orig_attrs = self.attributes.dup
+      args.each do |a|
+        self.attributes[a] = BSON::ObjectId(self.attributes[a])
+      end
+      res = yield
+      orig_attrs[:id] = self.attributes[:id]
+      self.attributes = orig_attrs
+      res
+    end
+    def self.for_dashboard(dashboard_id)
+      collection.find({:dashboard_id => BSON::ObjectId(dashboard_id.to_s)}).to_a.collect {|g| new(g) }
+    end
+  end
+end

data/lib/mouth/instrument.rb ADDED Viewed

@@ -0,0 +1,56 @@
+require 'socket'
+require 'benchmark'
+module Mouth
+  unless self.respond_to?(:measure)
+    class << self
+      attr_accessor :host, :port, :disabled
+      # Mouth.server = 'localhost:1234'
+      def server=(conn)
+        self.host, port = conn.split(':')
+        self.port = port.to_i
+      end
+      def host
+        @host || "localhost"
+      end
+      def port
+        @port || 8889
+      end
+      def measure(key, milli = nil)
+        result = nil
+        ms = milli || (Benchmark.realtime { result = yield } * 1000).to_i
+        write(key, ms, :ms)
+        result
+      end
+      def increment(key, delta = 1, sample_rate = nil)
+        write(key, delta, :c, sample_rate)
+      end
+      protected
+      def socket
+        @socket ||= UDPSocket.new
+      end
+      def write(k, v, op, sample_rate = nil)
+        return if self.disabled
+        if sample_rate
+          sample_rate = 1 if sample_rate > 1
+          return if rand > sample_rate
+        end
+        command = "#{k}:#{v}|#{op}"
+        command << "|@#{sample_rate}" if sample_rate
+        socket.send(command, 0, self.host, self.port)
+      end
+    end
+  end
+end

data/lib/mouth/record.rb ADDED Viewed

@@ -0,0 +1,72 @@
+module Mouth
+  class Record
+    # Keys are symbols
+    # id is :id, not _id
+    attr_accessor :attributes
+    def initialize(attrs = {})
+      self.attributes = normalize_attributes(attrs)
+    end
+    def all_attributes
+      self.attributes
+    end
+    def save
+      if self.attributes[:id]
+        attrs = self.attributes.dup
+        the_id = attrs.delete(:id).to_s
+        doc = self.class.collection.update({"_id" => BSON::ObjectId(the_id)}, attrs)
+      else
+        self.class.collection.insert(self.attributes)
+        self.attributes[:id] = self.attributes.delete(:_id).to_s
+      end
+      true
+    end
+    def update(new_attrs)
+      self.attributes = normalize_attributes(new_attrs)
+      self.save
+    end
+    def destroy
+      self.class.collection.remove({"_id" => BSON::ObjectId(self.attributes[:id])})
+    end
+    def normalize_attributes(attrs)
+      normalize = lambda do |h|
+        hd = {}
+        h.each_pair do |key, val|
+          val = normalize.call(val) if val.is_a?(Hash)
+          val = val.to_s if val.is_a?(BSON::ObjectId)
+          # TODO: arrays :(
+          hd[key.to_s == "_id" ? :id : key.to_sym] = val
+        end
+        hd
+      end
+      normalize.call attrs
+    end
+    def self.collection
+      demodularized = self.to_s.match(/(.+::)?(.+)$/)[2] || "record"
+      tableized = demodularized.downcase + "s" # (: lol :)
+      @collection ||= Mouth.mongo.collection(tableized)
+    end
+    def self.find(id)
+      collection.find({"_id" => BSON::ObjectId(id)}).to_a.collect {|d| new(d) }.first
+    end
+    def self.create(attributes)
+      r = new(attributes)
+      r.save
+      r
+    end
+    def self.all
+      collection.find.to_a.collect {|d| new(d) }
+    end
+  end
+end

data/lib/mouth/runner.rb ADDED Viewed

@@ -0,0 +1,89 @@
+require 'fileutils'
+require 'logger'
+require 'mouth/sucker'
+module Mouth
+  class Runner
+    attr_accessor :log_file
+    attr_accessor :pid_file
+    attr_accessor :logger
+    attr_accessor :verbosity    # 0: Only errors/warnings 1: informational 2: debug/all incomding UDP packets
+    attr_accessor :options
+    def initialize(opts={})
+      puts "Starting Mouth..."
+      self.log_file = opts[:log_file]
+      self.pid_file = opts[:pid_file]
+      self.verbosity = opts[:verbosity]
+      self.options = opts
+    end
+    def run!
+      kill! if self.options[:kill]
+      daemonize!
+      save_pid!
+      setup_logging!
+      # Start the reactor!
+      sucker = Mouth::Sucker.new(self.options)
+      sucker.suck!
+    end
+    def kill!
+      if @pid_file
+        pid = File.read(@pid_file)
+        #logger.warn "Sending #{kill_command} to #{pid.to_i}"
+        Process.kill(:INT, pid.to_i)
+      else
+        #logger.warn "No pid_file specified"
+      end
+    ensure
+      exit(0)
+    end
+    def daemonize!
+      # Fork and continue in forked process
+      # Also calls setsid
+      # Also redirects all output to /dev/null
+      Process.daemon(true)
+      # Reset umask
+      File.umask(0000)
+      # Set the procline
+      $0 = "mouth [initializing]"
+    end
+    def save_pid!
+      if @pid_file
+        pid = Process.pid
+        FileUtils.mkdir_p(File.dirname(@pid_file))
+        File.open(@pid_file, 'w') { |f| f.write(pid) }
+      end
+    end
+    def setup_logging!
+      if @log_file
+        STDERR.reopen(@log_file, 'a')
+        # Open a logger
+        self.logger = Logger.new(@log_file)
+        self.logger.level = case self.verbosity
+        when 0
+          Logger::WARN
+        when 1
+          Logger::INFO
+        else
+          Logger::DEBUG
+        end
+        Mouth.logger = self.logger
+        self.logger.info "Mouth Initialized..."
+      end
+    end
+  end # class Runner
+end # module Mouth

data/lib/mouth/sequence.rb ADDED Viewed

@@ -0,0 +1,284 @@
+module Mouth
+  # Usage:
+  # Sequence.new(["namespace.foobar_occurances"]).sequences
+  # # => {"foobar_occurances" => [4, 9, 0, ...]}
+  #
+  # Sequence.new(["namespace.foobar_occurances", "namespace.baz"], :kind => :timer).sequences
+  # # => {"foobar_occurances" => [{:count => 3, :min => 1, ...}, ...], "baz" => [...]}
+  #
+  # s = Sequence.new(...)
+  # s.time_sequence
+  # # => [Time.new(first datapoint), Time.new(second datapoint), ..., Time.new(last datapoint)]
+  class Sequence
+    attr_accessor :keys
+    attr_accessor :kind
+    attr_accessor :granularity_in_minutes
+    attr_accessor :start_time
+    attr_accessor :end_time
+    attr_accessor :namespace
+    attr_accessor :metrics
+    def initialize(keys, opts = {})
+      opts = {
+        :kind => :counter,
+        :granularity_in_minutes => 1,
+        :start_time => Time.now - (119 * 60),
+        :end_time => Time.now,
+      }.merge(opts)
+      self.keys = Array(keys)
+      self.kind = opts[:kind]
+      self.granularity_in_minutes = opts[:granularity_in_minutes]
+      self.start_time = opts[:start_time]
+      self.end_time = opts[:end_time]
+      self.metrics = []
+      namespaces = []
+      self.keys.each do |k|
+        namespace, metric = Mouth.parse_key(k)
+        namespaces << namespace
+        self.metrics << metric
+      end
+      raise StandardError.new("Batch calculation must come from the same namespace") if namespaces.uniq.length > 1
+      self.namespace = namespaces.first
+    end
+    def sequence
+      sequences.values.first
+    end
+    def sequences
+      return sequences_for_minute if self.granularity_in_minutes == 1
+      sequences_for_x_minutes(self.granularity_in_minutes)
+    end
+    def start_time_epoch
+      if self.granularity_in_minutes == 1
+        (self.start_time.to_i / 60) * 60
+      else
+        timestamp_to_nearest(self.start_time, self.granularity_in_minutes, :down) * 60
+      end
+    end
+    def time_sequence
+    end
+    # Epoch in seconds
+    def epoch_sequence
+    end
+    protected
+    def sequences_for_x_minutes(minutes)
+      start_timestamp = timestamp_to_nearest(self.start_time, minutes, :down)
+      end_timestamp = timestamp_to_nearest(self.end_time, minutes, :up)
+      # Then, for a timestamp xyz, it's in bucket (xyz - start_timestamp) / minutes
+      ###
+      map_function = <<-JS
+        function() {
+          emit(Math.floor((this.t - #{start_timestamp}) / #{minutes}), this);
+        }
+      JS
+      reduce_function = <<-JS
+        function(key, values) {
+          var result = {c: {}, m: {}}
+          ,   k
+          ,   existing
+          ,   current
+          ;
+          values.forEach(function(value) {
+            if (value.c) {
+              for (k in value.c) {
+                existing = result.c[k] || 0;
+                result.c[k] = existing + value.c[k]
+              }
+            }
+            if (value.m) {
+              for (k in value.m) {
+                current = value.m[k]
+                existing = result.m[k];
+                if (!existing) {
+                  current.median = [current.median];
+                  current.stddev = [current.stddev];
+                  current.mean = [current.mean];
+                  current.count = [current.count];
+                  result.m[k] = current;
+                } else {
+                  // {"count" => 0, "min" => nil, "max" => nil, "mean" => nil, "sum" => 0, "median" => nil, "stddev" => nil}
+                  // Ok here existing is a non null one of these ^, and so is current.  We just need to merge them.
+                  existing.min = existing.min < current.min ? existing.min : current.min;
+                  existing.max = existing.max > current.max ? existing.max : current.max;
+                  existing.sum = existing.sum + current.sum;
+                  // Save the individual stuff for later.  Need it later for proper merge.
+                  existing.median.push(current.median);
+                  existing.stddev.push(current.stddev);
+                  existing.mean.push(current.mean);
+                  existing.count.push(current.count);
+                }
+              }
+            }
+          });
+          for (k in result.m) {
+            existing = result.m[k];
+            var count = existing.median.length
+            ,   middle = Math.floor(count / 2)
+            ,   overallCount = 0
+            ,   secondMoment = 0
+            ,   mean
+            ,   i
+            ;
+            // Total datapoints
+            for (i = 0; i < count; i += 1) {
+              overallCount = overallCount + existing.count[i];
+            }
+            // Mean
+            mean = existing.sum / overallCount;
+            // Median: approximate and take the median median.
+            if (count % 2 == 0) {
+              existing.median = (existing.median[middle] + existing.median[middle - 1]) / 2;
+            } else {
+              existing.median = existing.median[middle];
+            }
+            // Stddev
+            // weighted average of "second moments": M2 += count(i)/overallCount * (stddev(i)^2 + mean(i)^2)
+            // Then stddev = sqrt(M2 - mean^2)
+            for (i = 0; i < count; i += 1) {
+              var stddev_i = existing.stddev[i]
+              ,   mean_i = existing.mean[i]
+              ;
+              secondMoment = secondMoment + (existing.count[i] / overallCount) * (stddev_i * stddev_i + mean_i * mean_i)
+            }
+            existing.stddev = Math.sqrt(secondMoment - mean * mean);
+            // Mean:
+            existing.mean = mean;
+            // Count
+            existing.count = overallCount;
+          }
+          return result;
+        }
+      JS
+      result = collection.map_reduce(map_function, reduce_function, :out => {:inline => true}, :raw => true, :query => {"t" => {"$gte" => start_timestamp, "$lte" => end_timestamp}})
+      docs = result["results"].collect do |r|
+        ordinal = r["_id"]
+        doc = r["value"]
+        doc["t"] = (start_timestamp + ordinal * minutes).to_i
+        doc
+      end
+      sequences_from_documents(docs, start_timestamp, end_timestamp - 1, minutes)
+    end
+    def sequences_for_minute
+      start_timestamp = self.start_time.to_i / 60
+      end_timestamp = self.end_time.to_i / 60
+      docs = self.collection.find({"t" => {"$gte" => start_timestamp, "$lte" => end_timestamp}}, :fields => self.fields).to_a
+      sequences_from_documents(docs, start_timestamp, end_timestamp, 1)
+    end
+    def sequences_from_documents(docs, start_timestamp, end_timestamp, minutes)
+      timestamp_to_metrics = docs.inject({}) do |h, e|
+        h[e["t"]] = e[self.kind_letter]
+        h
+      end
+      default = self.kind == :counter ? 0 : {"count" => 0, "min" => nil, "max" => nil, "mean" => nil, "sum" => 0, "median" => nil, "stddev" => nil}
+      seqs = {}
+      self.metrics.each do |m|
+        seq = []
+        (start_timestamp..end_timestamp).step(minutes) do |t|
+          mets = timestamp_to_metrics[t]
+          seq << ((mets && mets[m]) || default)
+        end
+        seqs[m] = seq
+      end
+      seqs
+    end
+    def collection
+      @collection ||= Mouth.collection(Mouth.mongo_collection_name(self.namespace))
+    end
+    def kind_letter
+      @kind_letter ||= self.kind == :counter ? "c" : "m"
+    end
+    def fields
+      @fields ||= ["t"].concat(self.metrics.map {|m| "#{kind_letter}.#{m}" })
+    end
+    # timestamp_to_nearest(Time.now, 15, :down)
+    # => t = 22122825 such that t * 60 is a second-epoch time on a 15-minute boundary, eg, 2012-01-23 17:45:00
+    def timestamp_to_nearest(time, minute, rounded = :down)
+      start_timestamp = time.to_i / 60 # This is minute granularity
+      if rounded == :down
+        start_timestamp -= time.min % minute
+      else
+        start_timestamp += minute - time.min % minute
+      end
+    end
+    public
+    # Generates a sample sequence of both counter and timing
+    def self.generate_sample(opts = {})
+      opts = {
+        :namespace => "sample",
+        :metric => "sample",
+        :start_time => (Time.now.to_i / 60 - 300),
+        :end_time => (Time.now.to_i / 60),
+      }.merge(opts)
+      collection_name = Mouth.mongo_collection_name(opts[:namespace])
+      counter = 99
+      (opts[:start_time]..opts[:end_time]).each do |t|
+        # Generate garbage data for the sample
+        # NOTE: candidate for improvement
+        m_count = rand(20) + 1
+        m_mean = rand(20) + 40
+        m_doc = {
+          "count" => m_count,
+          "min" => rand(20),
+          "max" => rand(20) + 80,
+          "mean" => m_mean,
+          "sum" => m_mean * m_count,
+          "median" => m_mean + rand(5),
+          "stddev" => rand(10)
+        }
+        # Insert the document into mongo
+        Mouth.collection(collection_name).update({"t" => t}, {"$set" => {"c.#{opts[:metric]}" => counter, "m.#{opts[:metric]}" => m_doc}}, :upsert => true)
+        # Update counter randomly
+        counter += rand(10) - 5
+        counter = 0 if counter < 0
+      end
+      true
+    end
+  end
+end