slurry 0.0.5 → 0.0.6

data/lib/slurry.rb

@@ -1,29 +1,89 @@
 require 'json'
 require "redis"
 require 'json2graphite'
-#require 'graphite-util'
 
+# @author Zach Leslie <zach@puppetlabs.com>
+#
 module Slurry
   module_function
 
-  # Pull in new data from STDIN
-  def funnel
+  # Handles connection details to the graphite server.
+  class Graphite
+
+    # Opens a socket with the specified graphite server.
+    #
+    # @param [String] server
+    # @param [String] port
+    def initialize(server,port)
+      @server, @port = server, port
+      @s = TCPSocket.open(server,port)
+    end
+
+    # Puts the graphite formatted string into the open socket.
+    #
+    # @param [String] target the graphite formatted target in dotted notion
+    # @param [String] value the value of the target
+    # @param [String] time the time that the sample was taken
+    def send (target,value,time)
+      line = [target,value,time].join(" ")
+      @s.puts(line)
+    end
+
+    # Closes the open socket to the graphite server.
+    #
+    def close
+      @s.close
+    end
+  end
+
+
+  # Wraps received hash in new hash with timestamp applied.
+  #
+  # @param [Hash] hash
+  # @parah [Int] time time of the recorded event
+  #
+  def timestamp (hash, time=Time.now.to_i)
     data = Hash.new
+    data[:data] = hash
+    data[:time] = time
+    data
+  end
 
+
+  # Reads from STDIN, expects json hash of just data.
+  # Creates new hash the original hash as the value of key :hash and the current unix time as key :time.  Calls method pipe() with resulting hash.
+  # {
+  #   :hash => { <hash read from STDIN> },
+  #   :time => <current unix time> 
+  #  }
+  #
+  def funnel
     body = ''
     body += STDIN.read
-    data[:hash] = JSON.parse(body)
-    data[:time] = Time.now.to_i
-    pipe(data)
-
+    jsondata = JSON.parse(body)
+    raise "jsondata is not of class Hash.  Is #{jsondata.class}" unless jsondata.is_a? Hash
+    pipe(timestamp(jsondata))
   end
 
-  # Push that data to redis
+  # Receives a hash formatted like so
+  #  {:time=>1345411779,
+  #  :collectd=>
+  #     {"myhostname"=>{"ntpd"=>-0.00142014}}}
+  #
+  # Pushes data into redis
   def pipe (hash)
     redis = Redis.new
     redis.lpush('slurry',hash.to_json)
   end
 
+  def push_to_redis (data, time=Time.now.to_i)
+    hash = Hash.new
+    hash[:data] = data
+    hash[:time] = time
+    r = Redis.new
+    r.lpush('slurry', hash.to_json)
+  end
+
   # Report the contents of the redis server
   def report
     r = Redis.new
@@ -32,7 +92,17 @@ module Slurry
     data[:slurry] = Hash.new
     data[:slurry][:waiting] = r.llen('slurry')
 
-    puts data.to_json
+    data
+  end
+
+  def inspect
+    r = Redis.new
+
+    data = Hash.new
+    data = r.lrange("slurry", 0, -1)
+
+    data
+
   end
 
   # Dump clean out everything from redis
@@ -45,25 +115,77 @@ module Slurry
 
   end
 
-  def liaise (server,port,wait=0.1)
+  def liaise (server,port,wait=0.01)
     r = Redis.new
 
     loop do
 
       # Pull something off the list
       popped = r.brpop('slurry')
-      data = JSON.parse(popped[1])
+      d = JSON.parse(popped)
+
+      raise "key 'data' not found in popped hash" unless d["data"]
+      raise "key 'time' not found in popped hash" unless d["time"]
 
       # Convert the json into graphite useable data
-      processed = Json2Graphite.get_graphite(data["hash"], data["time"])
-      s = TCPSocket.open(server, port)
-      processed.each do |line|
-        s.puts(line)
+      graphite = Json2Graphite.get_graphite(d["data"], d["time"])
+      graphite.each do |d|
+        target = d[:target].to_s
+        value  = d[:value].to_s
+        time   = d[:time].to_s
+        puts [target,value,time].join(' ')
       end
-      s.close
-      sleep wait
+      #sleep wait
     end
 
   end
 
+
+  def runonce (server,port,wait=0.1)
+
+    r = Redis.new           # open a new conection to redis
+    report = Hash.new       # initialize the report
+    report[:processed] = 0  # we've not processed any items yet
+
+    # open a socket with the graphite server
+    g = Slurry::Graphite.new(server,port)
+
+
+    # process every object in the list called 'slurry'
+    while r.llen('slurry') > 0 do
+
+      # pop the next object from the list
+      popped = r.rpop('slurry')
+      d = JSON.parse(popped)
+
+      # make syre the data we are about to use at least exists
+      raise "key 'data' not found in popped hash" unless d["data"]
+      raise "key 'time' not found in popped hash" unless d["time"]
+
+
+      # convert the object we popped into a graphite object
+      graphite = Json2Graphite.get_graphite(d["data"], d["time"])
+
+      # break the graphite object down into useable bits
+      graphite.each do |d|
+        # Make use of the values in the object
+        target = d[:target].to_s
+        value  = d[:value].to_s
+        time   = d[:time].to_s
+
+        # push the data to the open graphite socket
+        g.send(target,value, time)
+        # record the transaction
+        report[:processed] += 1
+      end
+      #sleep wait
+    end
+    # close up the connection to graphite
+    g.close
+
+    # return the report in json format
+    report.to_json
+
+  end
+
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: slurry
 version: !ruby/object:Gem::Version
-  version: 0.0.5
+  version: 0.0.6
   prerelease: 
 platform: ruby
 authors:
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-08-03 00:00:00.000000000 Z
+date: 2012-08-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json
@@ -59,7 +59,7 @@ dependencies:
     - - ! '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: A tool that caches json for graphite.
+description: A redis based collector for for data destined for graphite.
 email: xaque208@gmail.com
 executables:
 - slurry
@@ -67,8 +67,8 @@ extensions: []
 extra_rdoc_files: []
 files:
 - Rakefile
-- bin/liaise
 - bin/slurry
+- bin/liaise
 - lib/slurry.rb
 - etc/slurry.yaml.sample
 - README.md
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.24
+rubygems_version: 1.8.23
 signing_key: 
 specification_version: 3
 summary: A tool that caches json for graphite