slurry 0.0.5 → 0.0.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (2) hide show
  1. data/lib/slurry.rb +139 -17
  2. metadata +5 -5
data/lib/slurry.rb CHANGED
@@ -1,29 +1,89 @@
1
1
  require 'json'
2
2
  require "redis"
3
3
  require 'json2graphite'
4
- #require 'graphite-util'
5
4
 
5
+ # @author Zach Leslie <zach@puppetlabs.com>
6
+ #
6
7
  module Slurry
7
8
  module_function
8
9
 
9
- # Pull in new data from STDIN
10
- def funnel
10
+ # Handles connection details to the graphite server.
11
+ class Graphite
12
+
13
+ # Opens a socket with the specified graphite server.
14
+ #
15
+ # @param [String] server
16
+ # @param [String] port
17
+ def initialize(server,port)
18
+ @server, @port = server, port
19
+ @s = TCPSocket.open(server,port)
20
+ end
21
+
22
+ # Puts the graphite formatted string into the open socket.
23
+ #
24
+ # @param [String] target the graphite formatted target in dotted notion
25
+ # @param [String] value the value of the target
26
+ # @param [String] time the time that the sample was taken
27
+ def send (target,value,time)
28
+ line = [target,value,time].join(" ")
29
+ @s.puts(line)
30
+ end
31
+
32
+ # Closes the open socket to the graphite server.
33
+ #
34
+ def close
35
+ @s.close
36
+ end
37
+ end
38
+
39
+
40
+ # Wraps received hash in new hash with timestamp applied.
41
+ #
42
+ # @param [Hash] hash
43
+ # @parah [Int] time time of the recorded event
44
+ #
45
+ def timestamp (hash, time=Time.now.to_i)
11
46
  data = Hash.new
47
+ data[:data] = hash
48
+ data[:time] = time
49
+ data
50
+ end
12
51
 
52
+
53
+ # Reads from STDIN, expects json hash of just data.
54
+ # Creates new hash the original hash as the value of key :hash and the current unix time as key :time. Calls method pipe() with resulting hash.
55
+ # {
56
+ # :hash => { <hash read from STDIN> },
57
+ # :time => <current unix time>
58
+ # }
59
+ #
60
+ def funnel
13
61
  body = ''
14
62
  body += STDIN.read
15
- data[:hash] = JSON.parse(body)
16
- data[:time] = Time.now.to_i
17
- pipe(data)
18
-
63
+ jsondata = JSON.parse(body)
64
+ raise "jsondata is not of class Hash. Is #{jsondata.class}" unless jsondata.is_a? Hash
65
+ pipe(timestamp(jsondata))
19
66
  end
20
67
 
21
- # Push that data to redis
68
+ # Receives a hash formatted like so
69
+ # {:time=>1345411779,
70
+ # :collectd=>
71
+ # {"myhostname"=>{"ntpd"=>-0.00142014}}}
72
+ #
73
+ # Pushes data into redis
22
74
  def pipe (hash)
23
75
  redis = Redis.new
24
76
  redis.lpush('slurry',hash.to_json)
25
77
  end
26
78
 
79
+ def push_to_redis (data, time=Time.now.to_i)
80
+ hash = Hash.new
81
+ hash[:data] = data
82
+ hash[:time] = time
83
+ r = Redis.new
84
+ r.lpush('slurry', hash.to_json)
85
+ end
86
+
27
87
  # Report the contents of the redis server
28
88
  def report
29
89
  r = Redis.new
@@ -32,7 +92,17 @@ module Slurry
32
92
  data[:slurry] = Hash.new
33
93
  data[:slurry][:waiting] = r.llen('slurry')
34
94
 
35
- puts data.to_json
95
+ data
96
+ end
97
+
98
+ def inspect
99
+ r = Redis.new
100
+
101
+ data = Hash.new
102
+ data = r.lrange("slurry", 0, -1)
103
+
104
+ data
105
+
36
106
  end
37
107
 
38
108
  # Dump clean out everything from redis
@@ -45,25 +115,77 @@ module Slurry
45
115
 
46
116
  end
47
117
 
48
- def liaise (server,port,wait=0.1)
118
+ def liaise (server,port,wait=0.01)
49
119
  r = Redis.new
50
120
 
51
121
  loop do
52
122
 
53
123
  # Pull something off the list
54
124
  popped = r.brpop('slurry')
55
- data = JSON.parse(popped[1])
125
+ d = JSON.parse(popped)
126
+
127
+ raise "key 'data' not found in popped hash" unless d["data"]
128
+ raise "key 'time' not found in popped hash" unless d["time"]
56
129
 
57
130
  # Convert the json into graphite useable data
58
- processed = Json2Graphite.get_graphite(data["hash"], data["time"])
59
- s = TCPSocket.open(server, port)
60
- processed.each do |line|
61
- s.puts(line)
131
+ graphite = Json2Graphite.get_graphite(d["data"], d["time"])
132
+ graphite.each do |d|
133
+ target = d[:target].to_s
134
+ value = d[:value].to_s
135
+ time = d[:time].to_s
136
+ puts [target,value,time].join(' ')
62
137
  end
63
- s.close
64
- sleep wait
138
+ #sleep wait
65
139
  end
66
140
 
67
141
  end
68
142
 
143
+
144
+ def runonce (server,port,wait=0.1)
145
+
146
+ r = Redis.new # open a new conection to redis
147
+ report = Hash.new # initialize the report
148
+ report[:processed] = 0 # we've not processed any items yet
149
+
150
+ # open a socket with the graphite server
151
+ g = Slurry::Graphite.new(server,port)
152
+
153
+
154
+ # process every object in the list called 'slurry'
155
+ while r.llen('slurry') > 0 do
156
+
157
+ # pop the next object from the list
158
+ popped = r.rpop('slurry')
159
+ d = JSON.parse(popped)
160
+
161
+ # make syre the data we are about to use at least exists
162
+ raise "key 'data' not found in popped hash" unless d["data"]
163
+ raise "key 'time' not found in popped hash" unless d["time"]
164
+
165
+
166
+ # convert the object we popped into a graphite object
167
+ graphite = Json2Graphite.get_graphite(d["data"], d["time"])
168
+
169
+ # break the graphite object down into useable bits
170
+ graphite.each do |d|
171
+ # Make use of the values in the object
172
+ target = d[:target].to_s
173
+ value = d[:value].to_s
174
+ time = d[:time].to_s
175
+
176
+ # push the data to the open graphite socket
177
+ g.send(target,value, time)
178
+ # record the transaction
179
+ report[:processed] += 1
180
+ end
181
+ #sleep wait
182
+ end
183
+ # close up the connection to graphite
184
+ g.close
185
+
186
+ # return the report in json format
187
+ report.to_json
188
+
189
+ end
190
+
69
191
  end
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: slurry
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.0.5
4
+ version: 0.0.6
5
5
  prerelease:
6
6
  platform: ruby
7
7
  authors:
@@ -9,7 +9,7 @@ authors:
9
9
  autorequire:
10
10
  bindir: bin
11
11
  cert_chain: []
12
- date: 2012-08-03 00:00:00.000000000 Z
12
+ date: 2012-08-22 00:00:00.000000000 Z
13
13
  dependencies:
14
14
  - !ruby/object:Gem::Dependency
15
15
  name: json
@@ -59,7 +59,7 @@ dependencies:
59
59
  - - ! '>='
60
60
  - !ruby/object:Gem::Version
61
61
  version: '0'
62
- description: A tool that caches json for graphite.
62
+ description: A redis based collector for for data destined for graphite.
63
63
  email: xaque208@gmail.com
64
64
  executables:
65
65
  - slurry
@@ -67,8 +67,8 @@ extensions: []
67
67
  extra_rdoc_files: []
68
68
  files:
69
69
  - Rakefile
70
- - bin/liaise
71
70
  - bin/slurry
71
+ - bin/liaise
72
72
  - lib/slurry.rb
73
73
  - etc/slurry.yaml.sample
74
74
  - README.md
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
92
92
  version: '0'
93
93
  requirements: []
94
94
  rubyforge_project:
95
- rubygems_version: 1.8.24
95
+ rubygems_version: 1.8.23
96
96
  signing_key:
97
97
  specification_version: 3
98
98
  summary: A tool that caches json for graphite