hutils 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 6a480d2cd70d3539d12e0fcf0fbcdafa071df4a7
-  data.tar.gz: 13f269ea287bfc976fbcbcb60832a02374e21a49
+  metadata.gz: 083186100632ec210769f19bac9baa35d2353d3d
+  data.tar.gz: 1b43191d7f4711778b2100b4204d3e081522b8a0
 SHA512:
-  metadata.gz: 69a4a20b2ed69f2fe760d1e058dd6b30dfd62556cb559bbdd79a2518a680563436c483ff4ce168f7ce2b354453b224949725b789187d94567a5eb3233bf6b0b3
-  data.tar.gz: 5ed33ee6fc8a50e4c340f97e2f1d9c2a7c8d10e0c97c837ad20e64e8fac0c66933b751ed9176cbe6a6f85eb7e618451276c596a78423ef5eae0a00f026a2a969
+  metadata.gz: a89ffc10b37fda2ce243ba220bc75bf6993a30a2a1fa212fe95fad006153e5f524a9cf99cd7d367fceb7fd999cfc204b2fdcddd6fc9ee4bb0383504335373a94
+  data.tar.gz: 408f6904112389b780cc3b4d2efa695225c4bc486a005b16fa7201c8d6777a59e5c7927c1b26a4552bf379c07c9aded02ce392e407f473c2edc9718b7df7c692

data/lib/hutils/ltap/splunk_drainer.rb

@@ -33,7 +33,15 @@ module Hutils::Ltap
         end
       end
 
-      get_job_results
+      messages = []
+      loop do
+        new_messages = get_job_results(messages.count)
+        messages += new_messages
+        break if new_messages.count < MAX_RESULTS_PAGE
+      end
+
+      # give oldest first by default
+      messages.reverse
     end
 
     def cancel_job
@@ -50,12 +58,15 @@ module Hutils::Ltap
 
     private
 
+    MAX_RESULTS_PAGE = 100
+
     def create_job(query)
       resp = @api.post(
         path: "/servicesNS/#{@user}/search/search/jobs",
         expects: 201,
         body: URI.encode_www_form({
           earliest_time: @earliest.iso8601,
+          max_count: 10000,
           output_mode: "json",
           search: "search #{query}"
         })
@@ -81,19 +92,19 @@ module Hutils::Ltap
       debug("finalized")
     end
 
-    def get_job_results
+    def get_job_results(offset)
       # get results as CSV because the JSON version just mixes everything together
       # into a giant difficult-to-use blob
       resp = @api.get(
         path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}/results",
         # 204 if no results available
         expects: [200, 204],
-        body: URI.encode_www_form({
+        query: {
           action: "finalize",
-          # tell Splunk to give us all results
-          count: 0,
+          count: MAX_RESULTS_PAGE,
+          offset: offset,
           output_mode: "csv"
-        })
+        }
       )
 
       return [] if resp.status == 204
@@ -104,7 +115,7 @@ module Hutils::Ltap
       time_field = rows[0].index("_time") || raise("no _time field detected in Splunk response")
 
       # skip the first line as its used for CSV headers
-      rows[1..-1].
+      messages = rows[1..-1].
         map { |l| [l[raw_field], l[time_field]] }.
         # 2014-08-15T19:01:15.476590+00:00 54.197.117.24 local0.notice
         # api-web-1[23399]: - api.108080@heroku.com ...
@@ -112,9 +123,10 @@ module Hutils::Ltap
         map { |l, t| [l.strip, t] }.
         # format timestamps consistently (+00:00 --> Z)
         map { |l, t| [l, Time.parse(t).getutc.iso8601] }.
-        map { |l, t| @timestamps ? "#{t}: #{l}" : l }.
-        # results come in from newest to oldest; flip that
-        reverse
+        map { |l, t| @timestamps ? "#{t}: #{l}" : l }
+
+      debug("fetch results offset: #{offset}")
+      messages
     end
 
     def job_finished?
@@ -126,8 +138,12 @@ module Hutils::Ltap
         })
       )
       # Splunk may not be winning any awards for cleanest API anytime soon
-      state = JSON.parse(resp.body)["entry"][0]["content"]["dispatchState"]
-      debug("state: #{state}")
+      data = JSON.parse(resp.body)["entry"][0]["content"]
+      state = data["dispatchState"]
+      count, duration, state, ttl =
+        data.values_at("resultCount", "runDuration", "dispatchState", "ttl")
+      debug("result_count: #{count} run_duration: #{duration} " +
+        "state: #{state} ttl: #{ttl}")
       state == "DONE"
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hutils
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Brandur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-27 00:00:00.000000000 Z
+date: 2014-09-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon