slurry 0.0.5 → 0.0.6
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/lib/slurry.rb +139 -17
- metadata +5 -5
data/lib/slurry.rb
CHANGED
@@ -1,29 +1,89 @@
|
|
1
1
|
require 'json'
|
2
2
|
require "redis"
|
3
3
|
require 'json2graphite'
|
4
|
-
#require 'graphite-util'
|
5
4
|
|
5
|
+
# @author Zach Leslie <zach@puppetlabs.com>
|
6
|
+
#
|
6
7
|
module Slurry
|
7
8
|
module_function
|
8
9
|
|
9
|
-
#
|
10
|
-
|
10
|
+
# Handles connection details to the graphite server.
|
11
|
+
class Graphite
|
12
|
+
|
13
|
+
# Opens a socket with the specified graphite server.
|
14
|
+
#
|
15
|
+
# @param [String] server
|
16
|
+
# @param [String] port
|
17
|
+
def initialize(server,port)
|
18
|
+
@server, @port = server, port
|
19
|
+
@s = TCPSocket.open(server,port)
|
20
|
+
end
|
21
|
+
|
22
|
+
# Puts the graphite formatted string into the open socket.
|
23
|
+
#
|
24
|
+
# @param [String] target the graphite formatted target in dotted notion
|
25
|
+
# @param [String] value the value of the target
|
26
|
+
# @param [String] time the time that the sample was taken
|
27
|
+
def send (target,value,time)
|
28
|
+
line = [target,value,time].join(" ")
|
29
|
+
@s.puts(line)
|
30
|
+
end
|
31
|
+
|
32
|
+
# Closes the open socket to the graphite server.
|
33
|
+
#
|
34
|
+
def close
|
35
|
+
@s.close
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
|
40
|
+
# Wraps received hash in new hash with timestamp applied.
|
41
|
+
#
|
42
|
+
# @param [Hash] hash
|
43
|
+
# @parah [Int] time time of the recorded event
|
44
|
+
#
|
45
|
+
def timestamp (hash, time=Time.now.to_i)
|
11
46
|
data = Hash.new
|
47
|
+
data[:data] = hash
|
48
|
+
data[:time] = time
|
49
|
+
data
|
50
|
+
end
|
12
51
|
|
52
|
+
|
53
|
+
# Reads from STDIN, expects json hash of just data.
|
54
|
+
# Creates new hash the original hash as the value of key :hash and the current unix time as key :time. Calls method pipe() with resulting hash.
|
55
|
+
# {
|
56
|
+
# :hash => { <hash read from STDIN> },
|
57
|
+
# :time => <current unix time>
|
58
|
+
# }
|
59
|
+
#
|
60
|
+
def funnel
|
13
61
|
body = ''
|
14
62
|
body += STDIN.read
|
15
|
-
|
16
|
-
|
17
|
-
pipe(
|
18
|
-
|
63
|
+
jsondata = JSON.parse(body)
|
64
|
+
raise "jsondata is not of class Hash. Is #{jsondata.class}" unless jsondata.is_a? Hash
|
65
|
+
pipe(timestamp(jsondata))
|
19
66
|
end
|
20
67
|
|
21
|
-
#
|
68
|
+
# Receives a hash formatted like so
|
69
|
+
# {:time=>1345411779,
|
70
|
+
# :collectd=>
|
71
|
+
# {"myhostname"=>{"ntpd"=>-0.00142014}}}
|
72
|
+
#
|
73
|
+
# Pushes data into redis
|
22
74
|
def pipe (hash)
|
23
75
|
redis = Redis.new
|
24
76
|
redis.lpush('slurry',hash.to_json)
|
25
77
|
end
|
26
78
|
|
79
|
+
def push_to_redis (data, time=Time.now.to_i)
|
80
|
+
hash = Hash.new
|
81
|
+
hash[:data] = data
|
82
|
+
hash[:time] = time
|
83
|
+
r = Redis.new
|
84
|
+
r.lpush('slurry', hash.to_json)
|
85
|
+
end
|
86
|
+
|
27
87
|
# Report the contents of the redis server
|
28
88
|
def report
|
29
89
|
r = Redis.new
|
@@ -32,7 +92,17 @@ module Slurry
|
|
32
92
|
data[:slurry] = Hash.new
|
33
93
|
data[:slurry][:waiting] = r.llen('slurry')
|
34
94
|
|
35
|
-
|
95
|
+
data
|
96
|
+
end
|
97
|
+
|
98
|
+
def inspect
|
99
|
+
r = Redis.new
|
100
|
+
|
101
|
+
data = Hash.new
|
102
|
+
data = r.lrange("slurry", 0, -1)
|
103
|
+
|
104
|
+
data
|
105
|
+
|
36
106
|
end
|
37
107
|
|
38
108
|
# Dump clean out everything from redis
|
@@ -45,25 +115,77 @@ module Slurry
|
|
45
115
|
|
46
116
|
end
|
47
117
|
|
48
|
-
def liaise (server,port,wait=0.
|
118
|
+
def liaise (server,port,wait=0.01)
|
49
119
|
r = Redis.new
|
50
120
|
|
51
121
|
loop do
|
52
122
|
|
53
123
|
# Pull something off the list
|
54
124
|
popped = r.brpop('slurry')
|
55
|
-
|
125
|
+
d = JSON.parse(popped)
|
126
|
+
|
127
|
+
raise "key 'data' not found in popped hash" unless d["data"]
|
128
|
+
raise "key 'time' not found in popped hash" unless d["time"]
|
56
129
|
|
57
130
|
# Convert the json into graphite useable data
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
131
|
+
graphite = Json2Graphite.get_graphite(d["data"], d["time"])
|
132
|
+
graphite.each do |d|
|
133
|
+
target = d[:target].to_s
|
134
|
+
value = d[:value].to_s
|
135
|
+
time = d[:time].to_s
|
136
|
+
puts [target,value,time].join(' ')
|
62
137
|
end
|
63
|
-
|
64
|
-
sleep wait
|
138
|
+
#sleep wait
|
65
139
|
end
|
66
140
|
|
67
141
|
end
|
68
142
|
|
143
|
+
|
144
|
+
def runonce (server,port,wait=0.1)
|
145
|
+
|
146
|
+
r = Redis.new # open a new conection to redis
|
147
|
+
report = Hash.new # initialize the report
|
148
|
+
report[:processed] = 0 # we've not processed any items yet
|
149
|
+
|
150
|
+
# open a socket with the graphite server
|
151
|
+
g = Slurry::Graphite.new(server,port)
|
152
|
+
|
153
|
+
|
154
|
+
# process every object in the list called 'slurry'
|
155
|
+
while r.llen('slurry') > 0 do
|
156
|
+
|
157
|
+
# pop the next object from the list
|
158
|
+
popped = r.rpop('slurry')
|
159
|
+
d = JSON.parse(popped)
|
160
|
+
|
161
|
+
# make syre the data we are about to use at least exists
|
162
|
+
raise "key 'data' not found in popped hash" unless d["data"]
|
163
|
+
raise "key 'time' not found in popped hash" unless d["time"]
|
164
|
+
|
165
|
+
|
166
|
+
# convert the object we popped into a graphite object
|
167
|
+
graphite = Json2Graphite.get_graphite(d["data"], d["time"])
|
168
|
+
|
169
|
+
# break the graphite object down into useable bits
|
170
|
+
graphite.each do |d|
|
171
|
+
# Make use of the values in the object
|
172
|
+
target = d[:target].to_s
|
173
|
+
value = d[:value].to_s
|
174
|
+
time = d[:time].to_s
|
175
|
+
|
176
|
+
# push the data to the open graphite socket
|
177
|
+
g.send(target,value, time)
|
178
|
+
# record the transaction
|
179
|
+
report[:processed] += 1
|
180
|
+
end
|
181
|
+
#sleep wait
|
182
|
+
end
|
183
|
+
# close up the connection to graphite
|
184
|
+
g.close
|
185
|
+
|
186
|
+
# return the report in json format
|
187
|
+
report.to_json
|
188
|
+
|
189
|
+
end
|
190
|
+
|
69
191
|
end
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: slurry
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.0.
|
4
|
+
version: 0.0.6
|
5
5
|
prerelease:
|
6
6
|
platform: ruby
|
7
7
|
authors:
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2012-08-
|
12
|
+
date: 2012-08-22 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: json
|
@@ -59,7 +59,7 @@ dependencies:
|
|
59
59
|
- - ! '>='
|
60
60
|
- !ruby/object:Gem::Version
|
61
61
|
version: '0'
|
62
|
-
description: A
|
62
|
+
description: A redis based collector for for data destined for graphite.
|
63
63
|
email: xaque208@gmail.com
|
64
64
|
executables:
|
65
65
|
- slurry
|
@@ -67,8 +67,8 @@ extensions: []
|
|
67
67
|
extra_rdoc_files: []
|
68
68
|
files:
|
69
69
|
- Rakefile
|
70
|
-
- bin/liaise
|
71
70
|
- bin/slurry
|
71
|
+
- bin/liaise
|
72
72
|
- lib/slurry.rb
|
73
73
|
- etc/slurry.yaml.sample
|
74
74
|
- README.md
|
@@ -92,7 +92,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
92
92
|
version: '0'
|
93
93
|
requirements: []
|
94
94
|
rubyforge_project:
|
95
|
-
rubygems_version: 1.8.
|
95
|
+
rubygems_version: 1.8.23
|
96
96
|
signing_key:
|
97
97
|
specification_version: 3
|
98
98
|
summary: A tool that caches json for graphite
|