hutils 0.2.0 → 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/hutils/ltap/splunk_drainer.rb +28 -12
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 083186100632ec210769f19bac9baa35d2353d3d
|
|
4
|
+
data.tar.gz: 1b43191d7f4711778b2100b4204d3e081522b8a0
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a89ffc10b37fda2ce243ba220bc75bf6993a30a2a1fa212fe95fad006153e5f524a9cf99cd7d367fceb7fd999cfc204b2fdcddd6fc9ee4bb0383504335373a94
|
|
7
|
+
data.tar.gz: 408f6904112389b780cc3b4d2efa695225c4bc486a005b16fa7201c8d6777a59e5c7927c1b26a4552bf379c07c9aded02ce392e407f473c2edc9718b7df7c692
|
|
@@ -33,7 +33,15 @@ module Hutils::Ltap
|
|
|
33
33
|
end
|
|
34
34
|
end
|
|
35
35
|
|
|
36
|
-
|
|
36
|
+
messages = []
|
|
37
|
+
loop do
|
|
38
|
+
new_messages = get_job_results(messages.count)
|
|
39
|
+
messages += new_messages
|
|
40
|
+
break if new_messages.count < MAX_RESULTS_PAGE
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# give oldest first by default
|
|
44
|
+
messages.reverse
|
|
37
45
|
end
|
|
38
46
|
|
|
39
47
|
def cancel_job
|
|
@@ -50,12 +58,15 @@ module Hutils::Ltap
|
|
|
50
58
|
|
|
51
59
|
private
|
|
52
60
|
|
|
61
|
+
MAX_RESULTS_PAGE = 100
|
|
62
|
+
|
|
53
63
|
def create_job(query)
|
|
54
64
|
resp = @api.post(
|
|
55
65
|
path: "/servicesNS/#{@user}/search/search/jobs",
|
|
56
66
|
expects: 201,
|
|
57
67
|
body: URI.encode_www_form({
|
|
58
68
|
earliest_time: @earliest.iso8601,
|
|
69
|
+
max_count: 10000,
|
|
59
70
|
output_mode: "json",
|
|
60
71
|
search: "search #{query}"
|
|
61
72
|
})
|
|
@@ -81,19 +92,19 @@ module Hutils::Ltap
|
|
|
81
92
|
debug("finalized")
|
|
82
93
|
end
|
|
83
94
|
|
|
84
|
-
def get_job_results
|
|
95
|
+
def get_job_results(offset)
|
|
85
96
|
# get results as CSV because the JSON version just mixes everything together
|
|
86
97
|
# into a giant difficult-to-use blob
|
|
87
98
|
resp = @api.get(
|
|
88
99
|
path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}/results",
|
|
89
100
|
# 204 if no results available
|
|
90
101
|
expects: [200, 204],
|
|
91
|
-
|
|
102
|
+
query: {
|
|
92
103
|
action: "finalize",
|
|
93
|
-
|
|
94
|
-
|
|
104
|
+
count: MAX_RESULTS_PAGE,
|
|
105
|
+
offset: offset,
|
|
95
106
|
output_mode: "csv"
|
|
96
|
-
}
|
|
107
|
+
}
|
|
97
108
|
)
|
|
98
109
|
|
|
99
110
|
return [] if resp.status == 204
|
|
@@ -104,7 +115,7 @@ module Hutils::Ltap
|
|
|
104
115
|
time_field = rows[0].index("_time") || raise("no _time field detected in Splunk response")
|
|
105
116
|
|
|
106
117
|
# skip the first line as its used for CSV headers
|
|
107
|
-
rows[1..-1].
|
|
118
|
+
messages = rows[1..-1].
|
|
108
119
|
map { |l| [l[raw_field], l[time_field]] }.
|
|
109
120
|
# 2014-08-15T19:01:15.476590+00:00 54.197.117.24 local0.notice
|
|
110
121
|
# api-web-1[23399]: - api.108080@heroku.com ...
|
|
@@ -112,9 +123,10 @@ module Hutils::Ltap
|
|
|
112
123
|
map { |l, t| [l.strip, t] }.
|
|
113
124
|
# format timestamps consistently (+00:00 --> Z)
|
|
114
125
|
map { |l, t| [l, Time.parse(t).getutc.iso8601] }.
|
|
115
|
-
map { |l, t| @timestamps ? "#{t}: #{l}" : l }
|
|
116
|
-
|
|
117
|
-
|
|
126
|
+
map { |l, t| @timestamps ? "#{t}: #{l}" : l }
|
|
127
|
+
|
|
128
|
+
debug("fetch results offset: #{offset}")
|
|
129
|
+
messages
|
|
118
130
|
end
|
|
119
131
|
|
|
120
132
|
def job_finished?
|
|
@@ -126,8 +138,12 @@ module Hutils::Ltap
|
|
|
126
138
|
})
|
|
127
139
|
)
|
|
128
140
|
# Splunk may not be winning any awards for cleanest API anytime soon
|
|
129
|
-
|
|
130
|
-
|
|
141
|
+
data = JSON.parse(resp.body)["entry"][0]["content"]
|
|
142
|
+
state = data["dispatchState"]
|
|
143
|
+
count, duration, state, ttl =
|
|
144
|
+
data.values_at("resultCount", "runDuration", "dispatchState", "ttl")
|
|
145
|
+
debug("result_count: #{count} run_duration: #{duration} " +
|
|
146
|
+
"state: #{state} ttl: #{ttl}")
|
|
131
147
|
state == "DONE"
|
|
132
148
|
end
|
|
133
149
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: hutils
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.2.
|
|
4
|
+
version: 0.2.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Brandur
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2014-
|
|
11
|
+
date: 2014-09-04 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: excon
|