hutils 0.0.1 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 430d7ed6697d57adb45026682ae10d17bddd9e58
-  data.tar.gz: 229970d4107ed3f39c4b837aafc60d834eb46905
+  metadata.gz: 6a480d2cd70d3539d12e0fcf0fbcdafa071df4a7
+  data.tar.gz: 13f269ea287bfc976fbcbcb60832a02374e21a49
 SHA512:
-  metadata.gz: 43ea0dc9490a8f4fa51a719640e9d17afac0ca2d344aada8f9453e992ed23957b21e1ada30b415a2b7c7b7eed9c136fae9933079624998e87bfc16c86d9608db
-  data.tar.gz: 87aa2315323f14e683a96a9366c95189b2975530f3ab3ae9c7a465b3028b405b70727eed656deb508e9489abd44b519d69c93c4b88f9225eceb0b69b51106895
+  metadata.gz: 69a4a20b2ed69f2fe760d1e058dd6b30dfd62556cb559bbdd79a2518a680563436c483ff4ce168f7ce2b354453b224949725b789187d94567a5eb3233bf6b0b3
+  data.tar.gz: 5ed33ee6fc8a50e4c340f97e2f1d9c2a7c8d10e0c97c837ad20e64e8fac0c66933b751ed9176cbe6a6f85eb7e618451276c596a78423ef5eae0a00f026a2a969

data/README.md

@@ -8,6 +8,135 @@ A small collection of utilies for [logfmt](http://brandur.org/logfmt) processing
 gem install hutils
 ```
 
+## Utilities
+
+### lcut
+
+`lcut` extracts values from a logfmt trace based on some field name.
+
+```
+$ ltap 'instrumentation app=api earliest=-1m at=finish' | lcut method path
+GET     /providers/users/search
+GET     /vendor/resources/6307854
+GET     /health
+GET     /vendor/resources/6007506
+GET     /vendor/resources/7117492
+```
+
+### lfmt
+
+`lfmt` prettifies logfmt lines as they emerge from a stream, and highlights their key sections.
+
+(Note that the example below doesn't demonstrate color, which is one of the more important features of `logfmt`.)
+
+```
+$ ltap 'instrumentation app=api earliest=-1m at=finish' | lfmt
+api.108081@heroku.com app: api at: finish component: manager_apiauthorized elapsed: 0.008 instance_name: api.108081 instrumentation length: 339 method: GET path: /providers/users/search request_id: ef82825d-4c10-41f3-89ed-6bf805aa4513 status: 200 user: heroku-postgresql@addons.heroku.com user_id: 105750 version: 1
+api.136540@heroku.com app: api at: finish elapsed: 0.001 instance_name: api.136540 instrumentation method: GET path: /vendor/resources/6307854 request_id: 055df716-fc62-4554-b976-e2fe2472e107 status: 200 user: 3paccounts@dwnldmedia.com user_id: 97546 version: 2
+api.93579@heroku.com app: api at: finish elapsed: 0.000 instance_name: api.93579 instrumentation method: GET path: /health request_id: 6af07088-82af-4f50-87c1-8b5d248807f0 status: 200 user: heroku-postgresql@addons.heroku.com user_id: 105750 version: 1
+api.108081@heroku.com app: api at: finish elapsed: 0.174 instance_name: api.108081 instrumentation method: GET path: /vendor/resources/6007506 request_id: ef82825d-4c10-41f3-89ed-6bf805aa4513 status: 200 user: heroku-postgresql@addons.heroku.com user_id: 105750 version: 1
+api.108081@heroku.com app: api at: finish elapsed: 0.162 instance_name: api.108081 instrumentation method: GET path: /vendor/resources/7117492 request_id: 7480424d-5a8a-488a-a32a-55812fde5f4b status: 200 user: heroku-postgresql@addons.heroku.com user_id: 105750 version: 1
+```
+
+### ltap
+
+`ltap` accesses messages from popular log providers in a consistent way so that it can easily be parsed by other utilities that operate on logfmt traces. Currently supported providers are Papertrail and Splunk.
+
+```
+$ ltap 'instrumentation app=api earliest=-1m at=finish'
+api.108081@heroku.com instrumentation method=GET path=/providers/users/search request_id=d5c373fd-d1ec-4986-bc43-2617431116f2 at=finish elapsed=0.008 length=339 status=200 app=api instance_name=api.108081 version=1 component=manager_apiauthorized app=api instance_name=api.108081 request_id=ef82825d-4c10-41f3-89ed-6bf805aa4513 version=1 user=heroku-postgresql@addons.heroku.com user_id=105750
+api.136540@heroku.com instrumentation method=GET path=/vendor/resources/6307854 request_id=d2f25032-9aaa-41e9-8aaf-9a46a44523d1 at=finish elapsed=0.110 status=200 app=api instance_name=api.136540 version=1 user=heroku-postgresql@addons.heroku.com user_id=105750step=check_oauth_scope! request_id=055df716-fc62-4554-b976-e2fe2472e107 version=2 user=account@example.com user_id=97546 app=api instance_name=api.136540 at=finish elapsed=0.001
+api.93579@heroku.com instrumentation method=GET path=/health request_id=6af07088-82af-4f50-87c1-8b5d248807f0 at=finish elapsed=0.000 status=200 app=api instance_name=api.93579 version=1 user=heroku-postgresql@addons.heroku.com user_id=105750
+api.108081@heroku.com instrumentation method=GET path=/vendor/resources/6007506 request_id=ef82825d-4c10-41f3-89ed-6bf805aa4513 at=finish elapsed=0.174 status=200 app=api instance_name=api.108081 version=1 user=heroku-postgresql@addons.heroku.com user_id=105750
+api.108081@heroku.com instrumentation method=GET path=/vendor/resources/7117492 request_id=7480424d-5a8a-488a-a32a-55812fde5f4b at=finish elapsed=0.162 status=200 app=api instance_name=api.108081 version=1 user=heroku-postgresql@addons.heroku.com user_id=105750
+```
+
+`ltap` can be configured using `~/.ltap`. A sample Papertrail configuration looks like the following:
+
+```
+[global]
+profile = my_papertrail
+
+[my_papertrail]
+key = an-api-key
+type = papertrail
+```
+
+A sample Splunk configuration:
+
+```
+[global]
+profile = my_splunk
+
+[my_splunk]
+earliest = -24h
+type = splunk
+url = https://brandur:an-api-key@splunk.example.com:8089
+```
+
+### lviz
+
+`lviz` helps to visualize logfmt output by building a tree out of some set of data by combining common sets of key/value pairs into shared parent nodes. Messages remain ordered by time, which removes some potential for commonality, but in many cases a disproportionate number of attributes can be moved up to nodes close to the top of the tree. Output is colorized and important keys are highlighted to make traces more easily digestible.
+
+```
+$ ltap 'instrumentation app=api earliest=-1m at=finish' | lviz
++ app: api
+  instrumentation
+  method: GET
+  status: 200
+
+        + api.108081@heroku.com
+          component: manager_apiauthorized
+          elapsed: 0.008
+          instance_name: api.108081
+          length: 339
+          path: /providers/users/search
+          request_id: ef82825d-4c10-41f3-89ed-6bf805aa4513
+          user: heroku-postgresql@addons.heroku.com
+          user_id: 105750
+          version: 1
+
+        + api.136540@heroku.com
+          elapsed: 0.001
+          instance_name: api.136540
+          path: /vendor/resources/6307854
+          request_id: 055df716-fc62-4554-b976-e2fe2472e107
+          user: account@example.com
+          user_id: 97546
+          version: 2
+
+        + user: heroku-postgresql@addons.heroku.com
+          user_id: 105750
+          version: 1
+
+                + api.93579@heroku.com
+                  elapsed: 0.000
+                  instance_name: api.93579
+                  path: /health
+                  request_id: 6af07088-82af-4f50-87c1-8b5d248807f0
+
+                + api.108081@heroku.com
+                  instance_name: api.108081
+
+                        + elapsed: 0.174
+                          path: /vendor/resources/6007506
+                          request_id: ef82825d-4c10-41f3-89ed-6bf805aa4513
+
+                        + elapsed: 0.162
+                          path: /vendor/resources/7117492
+                          request_id: 7480424d-5a8a-488a-a32a-55812fde5f4b
+```
+
+`lviz` can be configured with `~/.lviz`. For example:
+
+```
+[global]
+highlights = path,user
+ignore = at
+```
+
+`lviz` can also produce a compact mode of output using `-c` or `--compact`.
+
 ## Testing
 
 ```

data/bin/lcut

@@ -32,8 +32,8 @@ end
 
 file = file ? File.open(file) : $stdin
 file.each_line do |line|
-  messages = Hutils::Parser.new(line).parse
-  messages.each do |message|
+  events = Hutils::Parser.new(line).parse
+  events.each do |message, _|
     values = ARGV.map { |f| message[f] ? message[f] : nil }
     if allow_empty || !values.all? { |v| v.nil? }
       puts values.join(delimiter)

data/bin/lfmt

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+
+require_relative "../lib/hutils"
+
+colors = $stdout.tty?
+highlights = %w(endpoint path route_signature user user_id)
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: lsel [options] <fields>"
+  opts.on("-d", "--delimiter [DELIMITER]", "Delimiter separating output") { |d|
+    delimiter = d
+  }
+  opts.on("-e", "--allow-empty", "Output empty lines") { |e| allow_empty = e }
+  opts.on("-f", "--file [FILE]", "File to read") { |f| file = f }
+  opts.on("-h", "--help", "Show this help string") { |h|
+    if h
+      puts(opts.help)
+      exit(0)
+    end
+  }
+  opts.on("-v", "--verbose", "Verbose mode") { |v| verbose = v }
+end
+opts.parse!
+
+line_visualizer = Hutils::TextVisualizer::LineVisualizer.new(
+  colors: colors,
+  compact: true,
+  highlights: highlights,
+  out: $stdout)
+
+ARGF.each_line do |line|
+  events = Hutils::Parser.new(line).parse
+  events.each do |message, time|
+    line_visualizer.display(message, time: time)
+  end
+end

data/bin/ltap

@@ -16,9 +16,13 @@ opts = OptionParser.new do |opts|
       exit(0)
     end
   }
+  opts.on("-e", "--earliest [TIME]", "Bound for earliest message") { |e|
+    conf.earliest = e
+  }
   opts.on("-k", "--key [KEY]", "Service API key") { |t| conf.timeout = t }
   opts.on("-p", "--profile [PROFILE]", "Conf profile") { |p| conf.profile = p }
   opts.on("-t", "--timeout [TIMEOUT]", "Job timeout") { |t| conf.timeout = t }
+  opts.on("-s", "--timestamps", "Output timestamps") { |t| conf.timestamps = t }
   opts.on("-y", "--type [TYPE]", "Service type") { |t| conf.timeout = t }
   opts.on("-u", "--url [URL]", "Service API URL") { |u| conf.url = u }
   opts.on("-v", "--verbose", "Verbose mode") { |v| conf.verbose = v }
@@ -38,6 +42,13 @@ unless ARGV.first
   abort(opts.help)
 end
 
+parser = Hutils::Ltap::TimeBoundParser.new
+if !(earliest = parser.parse(conf.earliest, from: Time.now.getutc))
+  $stderr.puts %{Couldn't parse bound "#{conf.earliest}", reverting to default}
+  # duplicated from conf, which is not ideal
+  earliest = Time.now.getutc - 60 * 60 * 24
+end
+
 drainer = case conf.type
 when "papertrail"
   Hutils::Ltap::PaperTrailDrainer
@@ -48,9 +59,11 @@ else
 end
 
 drainer = drainer.new(
+  earliest: earliest,
   key: conf.key,
   query: ARGV.first,
   timeout: conf.timeout,
+  timestamps: conf.timestamps,
   url: conf.url,
   verbose: conf.verbose
 )

data/bin/lviz

@@ -7,7 +7,7 @@ require_relative "../lib/hutils"
 colors = $stdout.tty?
 compact = false
 highlights = %w(endpoint path route_signature user user_id)
-ignore = %w(at)
+ignore = %w()
 interactive = false
 
 ini = IniFile.load(ENV["HOME"] + "/.lviz")
@@ -34,15 +34,16 @@ opts = OptionParser.new do |opts|
     ignore = i.split(",") rescue []
   }
   opts.on("-i", "--interactive", "Interactive mode") { |i| interactive = i }
-  opts.on("--no-color", "Disable colors") { |c| colors = c }
+  opts.on("--no-color", "Disable colors") { |c| colors = false }
 end
 opts.parse!
 
 # ARGF will read from stdin if available, and otherwise fall back to files
 # named as parameters (similar to the behavior of `cat`)
-lines = Hutils::Parser.new(ARGF.read).parse
-Hutils::Stripper.new(lines, ignore).run
-root = Hutils::TreeBuilder.new(lines).build
+events = Hutils::Parser.new(ARGF.read).parse
+messages = events.map { |e| e[0] }
+Hutils::Stripper.new(messages, ignore).run
+root = Hutils::TreeBuilder.new(messages).build
 
 if interactive
   require_relative "../lib/hutils/curses_visualizer"
@@ -59,8 +60,6 @@ if interactive
     root: root
   ).run
 else
-  require_relative "../lib/hutils/text_visualizer"
-
   Hutils::TextVisualizer.new(
     colors: colors,
     compact: compact,

data/lib/hutils/ltap/conf.rb

@@ -2,16 +2,20 @@ require "inifile"
 
 module Hutils::Ltap
   class Conf
+    attr_accessor :earliest
     attr_accessor :key
     attr_accessor :profile
     attr_accessor :timeout
+    attr_accessor :timestamps
     attr_accessor :type
     attr_accessor :url
     attr_accessor :verbose
 
     def initialize
       @ini = IniFile.load(ENV["HOME"] + "/.ltap")
+      self.earliest = "-24h"
       self.timeout = 60
+      self.timestamps = false
       self.verbose = false
     end
 
@@ -21,9 +25,11 @@ module Hutils::Ltap
 
     def load_section(name)
       if section = @ini && @ini[name]
+        load_value(section, :earliest)
         load_value(section, :key)
         load_value(section, :profile)
         load_value(section, :timeout)
+        load_value(section, :timestamps)
         load_value(section, :type)
         load_value(section, :url)
         load_value(section, :verbose)

data/lib/hutils/ltap/paper_trail_drainer.rb

@@ -4,13 +4,15 @@ module Hutils::Ltap
   class PaperTrailDrainer
     PAPER_TRAIL_URL = "https://papertrailapp.com"
 
-    def initialize(key:, timeout:, query:, url:, verbose:)
+    def initialize(earliest:, key:, timeout:, query:, timestamps:, url:, verbose:)
       @api = Excon.new(PAPER_TRAIL_URL,
         headers: {
           "X-Papertrail-Token" => key
         })
+      @earliest = earliest
       @query = query
       @timeout = timeout
+      @timestamps = timestamps
       @verbose = verbose
     end
 
@@ -20,12 +22,25 @@ module Hutils::Ltap
       start = Time.now
 
       loop do
-        new_messages, reached_beginning, min_id = fetch_page(min_id)
+        new_messages, reached_beginning, min_id, min_time = fetch_page(min_id)
         messages += new_messages
 
         # break if PaperTrail has indicated that we've reached the beginning of
-        # our results, or if we've approximately hit our timeout
-        if reached_beginning || (Time.now - start).to_i > @timeout
+        # our results
+        if reached_beginning
+          debug("breaking: reached beginning")
+          break
+        end
+
+        # or if we've reached back before our earliest
+        if min_time && min_time < @earliest
+          debug("breaking: before earliest: #{@earliest}")
+          break
+        end
+
+        # or if we've approximately hit our timeout
+        if (Time.now - start).to_i > @timeout
+          debug("breaking: reached timeout")
           break
         end
       end
@@ -45,6 +60,16 @@ module Hutils::Ltap
     class RateLimited < StandardError
     end
 
+    def build_message(event)
+      message = event["message"].strip
+      if @timestamps
+        # it's already in ISO8601, but let's make it UTC
+        t = Time.parse(event["received_at"]).getutc.iso8601
+        message = "#{t}: #{message}"
+      end
+      message
+    end
+
     def debug(str)
       if @verbose
         puts str
@@ -65,11 +90,17 @@ module Hutils::Ltap
       end
 
       data = JSON.parse(resp.body)
+      events = data["events"]
       debug("backend_timeout: #{data["backend_timeout"] || false} " +
         "min_id: #{data["min_id"]} " +
         "reached_beginning: #{data["reached_beginning"] || false}")
-      messages = data["events"].map { |e| e["message"].strip }
-      [messages, data["reached_beginning"], data["min_id"]]
+
+      [
+        events.map { |e| build_message(e) },
+        data["reached_beginning"],
+        data["min_id"],
+        events.last ? Time.parse(events.last["received_at"]) : nil
+      ]
     end
   end
 end

data/lib/hutils/ltap/splunk_drainer.rb

@@ -1,13 +1,16 @@
 require "csv"
 require "excon"
 require "json"
+require "time"
 require "uri"
 
 module Hutils::Ltap
   class SplunkDrainer
-    def initialize(key:, timeout:, query:, url:, verbose:)
+    def initialize(earliest:, key:, timeout:, query:, timestamps:, url:, verbose:)
+      @earliest = earliest
       @timeout = timeout
       @query = query
+      @timestamps = timestamps
       @verbose = verbose
 
       @user = URI.parse(url).user
@@ -24,6 +27,7 @@ module Hutils::Ltap
 
         # finalize the job if we've broken our timeout point
         if (Time.now - start).to_i > @timeout
+          debug("breaking: reached timeout")
           finalize_job
           break
         end
@@ -51,6 +55,7 @@ module Hutils::Ltap
         path: "/servicesNS/#{@user}/search/search/jobs",
         expects: 201,
         body: URI.encode_www_form({
+          earliest_time: @earliest.iso8601,
           output_mode: "json",
           search: "search #{query}"
         })
@@ -85,6 +90,8 @@ module Hutils::Ltap
         expects: [200, 204],
         body: URI.encode_www_form({
           action: "finalize",
+          # tell Splunk to give us all results
+          count: 0,
           output_mode: "csv"
         })
       )
@@ -93,15 +100,19 @@ module Hutils::Ltap
 
       rows = CSV.parse(resp.body)
       return [] if rows.count < 1
-      field = rows[0].index("_raw") || raise("no _raw field detected in Splunk response")
+      raw_field = rows[0].index("_raw") || raise("no _raw field detected in Splunk response")
+      time_field = rows[0].index("_time") || raise("no _time field detected in Splunk response")
 
       # skip the first line as its used for CSV headers
       rows[1..-1].
-        map { |l| l[field] }.
+        map { |l| [l[raw_field], l[time_field]] }.
         # 2014-08-15T19:01:15.476590+00:00 54.197.117.24 local0.notice
         # api-web-1[23399]: - api.108080@heroku.com ...
-        map { |l| l.gsub(/^.*: - /, "") }.
-        map { |l| l.strip }.
+        map { |l, t| [l.gsub(/^.*: - /, ""), t] }.
+        map { |l, t| [l.strip, t] }.
+        # format timestamps consistently (+00:00 --> Z)
+        map { |l, t| [l, Time.parse(t).getutc.iso8601] }.
+        map { |l, t| @timestamps ? "#{t}: #{l}" : l }.
         # results come in from newest to oldest; flip that
         reverse
     end

data/lib/hutils/ltap/time_bound_parser.rb

@@ -0,0 +1,41 @@
+module Hutils::Ltap
+  # A parser designed to support the same style of relative time that Splunk
+  # does, but note that absolute dates and "snap to" times are not supported.
+  #
+  # See here for more information on this format:
+  #
+  #      http://docs.splunk.com/Documentation/Splunk/6.1.3/SearchReference/SearchTimeModifiers
+  class TimeBoundParser
+    def parse(str, from: Time.now.getutc)
+      if str =~ /^([+\-])([0-9]+)?([a-z]+)$/
+        to_date($2 || 1, $3, from: from)
+      else
+        nil
+      end
+    end
+
+    private
+
+    def to_date(num, unit, from: from)
+      unit_time = case unit
+      when "s", "sec", "secs", "second", "seconds"
+        1
+      when "m", "min", "minute", "minutes"
+        60
+      when "h", "hr", "hrs", "hour", "hours"
+        60 * 60
+      when "d", "day", "days"
+        60 * 60 * 24
+      when "w", "week", "weeks"
+        60 * 60 * 24 * 7
+      when "mon", "month", "months"
+        60 * 60 * 24 * 7 * 30
+      when "q", "qtr", "qtrs", "quarter", "quarters"
+        60 * 60 * 24 * 7 * 30 * 3
+      when "y", "yr", "yrs", "year", "years"
+        60 * 60 * 24 * 7 * 30 * 365
+      end
+      from - num.to_f * unit_time
+    end
+  end
+end

data/lib/hutils/ltap.rb

@@ -1,3 +1,4 @@
 require_relative "ltap/conf"
 require_relative "ltap/paper_trail_drainer"
 require_relative "ltap/splunk_drainer"
+require_relative "ltap/time_bound_parser"

data/lib/hutils/text_visualizer.rb

@@ -1,8 +1,58 @@
 require "term/ansicolor"
+require "time"
 
 module Hutils
   class TextVisualizer
-    include Term::ANSIColor
+    class LineVisualizer
+      include Term::ANSIColor
+
+      def initialize(colors:, compact:, highlights:, out:)
+        @colors = colors
+        @compact = compact
+        @highlights = highlights
+        @out = out
+      end
+
+      def display(message, indent: "", time: nil)
+        if time
+          @out.print "#{colorize(:cyan, time.iso8601)} "
+        end
+        message.to_a.sort_by { |k, v| k }.map { |k, v|
+          pair_to_string(k, v)
+        }.each_with_index { |display, i|
+          if @compact
+            @out.print(indent) if i == 0
+            @out.print("#{display} ")
+          else
+            marker = i == 0 ? "+ " : "  "
+            @out.puts "#{indent}#{marker}#{display}"
+          end
+        }
+        @out.puts ""
+      end
+
+      private
+
+      def colorize(method, str)
+        if @colors
+          send(method, str)
+        else
+          str
+        end
+      end
+
+      def pair_to_string(k, v)
+        if v == true
+          colorize(:green, k)
+        else
+          if @highlights.include?(k)
+            colorize(:on_yellow, colorize(:black, "#{k}: #{v}"))
+          else
+            "#{colorize(:green, k)}: #{v}"
+          end
+        end
+      end
+    end
 
     def initialize(colors:, compact:, highlights:, root:, out:)
       @colors = colors
@@ -10,52 +60,28 @@ module Hutils
       @highlights = highlights
       @out = out
       @root = root
+
+      @line_visualizer = LineVisualizer.new(
+        colors: colors,
+        compact: compact,
+        highlights: highlights,
+        out: out
+      )
     end
 
     def display
       display_node(@root)
     end
 
-    private 
-
-    def colorize(method, str)
-      if @colors
-        send(method, str)
-      else
-        str
-      end
-    end
+    private
 
     def display_node(node)
       if !node.common.empty?
         # the "- 1" is because the root node is empty
         indent = "\t" * (node.depth - 1)
-        node.common.to_a.sort_by { |k, v| k }.map { |k, v|
-          pair_to_string(k, v)
-        }.each_with_index { |display, i|
-          if @compact
-            @out.print(indent) if i == 0
-            @out.print("#{display} ")
-          else
-            marker = i == 0 ? "+ " : "  "
-            @out.puts "#{indent}#{marker}#{display}"
-          end
-        }
-        @out.puts ""
+        @line_visualizer.display(node.common, indent: indent)
       end
       node.slots.each { |slot| display_node(slot) }
     end
-
-    def pair_to_string(k, v)
-      if v == true
-        colorize(:green, k)
-      else
-        if @highlights.include?(k)
-          colorize(:on_yellow, colorize(:black, "#{k}: #{v}"))
-        else
-          "#{colorize(:green, k)}: #{v}"
-        end
-      end
-    end
   end
 end

data/lib/hutils.rb

@@ -1,4 +1,7 @@
+require "time"
+
 require_relative "hutils/stripper"
+require_relative "hutils/text_visualizer"
 
 module Hutils
   class Node
@@ -63,16 +66,16 @@ module Hutils
     end
 
     def parse
-      lines = @str.split("\n").map { |line| normalize(line) }
-      lines.map! do |line|
-        pairs = line.scan(/(?:['"](?:\\.|[^'"])*['"]|[^'" ])+/).map do |pair|
+      events = @str.split("\n").map { |line| normalize(line) }
+      events.map! do |message, time|
+        pairs = message.scan(/(?:['"](?:\\.|[^'"])*['"]|[^'" ])+/).map do |pair|
           key, value = pair.split("=")
           [key, value].each do |str|
             str.gsub!(/^['"]?(.*?)['"]?$/, '\1') if str
           end
           [key, value || true]
         end
-        Hash[pairs]
+        [Hash[pairs], time]
       end
     end
 
@@ -80,7 +83,8 @@ module Hutils
 
     def normalize(line)
       line = line.strip
-      line.gsub(/^[T0-9\-:+.]+( [a-z]+\[[a-z0-9\-_.]+\])?: /, '')
+      line = line.gsub(/^([TZ0-9\-:+.]+)( [a-z]+\[[a-z0-9\-_.]+\])?: /, '')
+      [line, $1 ? Time.parse($1).getutc : nil]
     end
   end
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: hutils
 version: !ruby/object:Gem::Version
-  version: 0.0.1
+  version: 0.2.0
 platform: ruby
 authors:
 - Brandur
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-08-22 00:00:00.000000000 Z
+date: 2014-08-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: excon
@@ -74,6 +74,7 @@ description:
 email: brandur@mutelight.org
 executables:
 - lcut
+- lfmt
 - ltap
 - lviz
 extensions: []
@@ -85,11 +86,13 @@ files:
 - "./lib/hutils/ltap/conf.rb"
 - "./lib/hutils/ltap/paper_trail_drainer.rb"
 - "./lib/hutils/ltap/splunk_drainer.rb"
+- "./lib/hutils/ltap/time_bound_parser.rb"
 - "./lib/hutils/node_navigator.rb"
 - "./lib/hutils/stripper.rb"
 - "./lib/hutils/text_visualizer.rb"
 - README.md
 - bin/lcut
+- bin/lfmt
 - bin/ltap
 - bin/lviz
 homepage: https://github.com/brandur/hutils