hutils 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 430d7ed6697d57adb45026682ae10d17bddd9e58
+  data.tar.gz: 229970d4107ed3f39c4b837aafc60d834eb46905
+SHA512:
+  metadata.gz: 43ea0dc9490a8f4fa51a719640e9d17afac0ca2d344aada8f9453e992ed23957b21e1ada30b415a2b7c7b7eed9c136fae9933079624998e87bfc16c86d9608db
+  data.tar.gz: 87aa2315323f14e683a96a9366c95189b2975530f3ab3ae9c7a465b3028b405b70727eed656deb508e9489abd44b519d69c93c4b88f9225eceb0b69b51106895

data/README.md

@@ -0,0 +1,16 @@
+# hutils
+
+A small collection of utilies for [logfmt](http://brandur.org/logfmt) processing.
+
+## Installation
+
+```
+gem install hutils
+```
+
+## Testing
+
+```
+bundle install
+bundle exec rake
+```

data/bin/lcut

@@ -0,0 +1,43 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+
+require_relative "../lib/hutils"
+
+allow_empty = false
+delimiter = "\t"
+file = nil
+verbose = false
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: lsel [options] <fields>"
+  opts.on("-d", "--delimiter [DELIMITER]", "Delimiter separating output") { |d|
+    delimiter = d
+  }
+  opts.on("-e", "--allow-empty", "Output empty lines") { |e| allow_empty = e }
+  opts.on("-f", "--file [FILE]", "File to read") { |f| file = f }
+  opts.on("-h", "--help", "Show this help string") { |h|
+    if h
+      puts(opts.help)
+      exit(0)
+    end
+  }
+  opts.on("-v", "--verbose", "Verbose mode") { |v| verbose = v }
+end
+opts.parse!
+
+if !file && $stdin.tty? || !ARGV.first
+  abort(opts.help)
+end
+
+file = file ? File.open(file) : $stdin
+file.each_line do |line|
+  messages = Hutils::Parser.new(line).parse
+  messages.each do |message|
+    values = ARGV.map { |f| message[f] ? message[f] : nil }
+    if allow_empty || !values.all? { |v| v.nil? }
+      puts values.join(delimiter)
+    end
+  end
+end
+file.close

data/bin/ltap

@@ -0,0 +1,70 @@
+#!/usr/bin/env ruby
+
+require "optparse"
+
+require_relative "../lib/hutils"
+require_relative "../lib/hutils/ltap"
+
+conf = Hutils::Ltap::Conf.new
+conf.load
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: ltap [options] <query>"
+  opts.on("-h", "--help", "Show this help string") { |h|
+    if h
+      puts(opts.help)
+      exit(0)
+    end
+  }
+  opts.on("-k", "--key [KEY]", "Service API key") { |t| conf.timeout = t }
+  opts.on("-p", "--profile [PROFILE]", "Conf profile") { |p| conf.profile = p }
+  opts.on("-t", "--timeout [TIMEOUT]", "Job timeout") { |t| conf.timeout = t }
+  opts.on("-y", "--type [TYPE]", "Service type") { |t| conf.timeout = t }
+  opts.on("-u", "--url [URL]", "Service API URL") { |u| conf.url = u }
+  opts.on("-v", "--verbose", "Verbose mode") { |v| conf.verbose = v }
+end
+opts.parse!
+
+# load a profile from ~/.ltap if one was specified
+if conf.profile
+  conf.load_section(conf.profile)
+end
+
+unless conf.type
+  abort("Must set a service type; use ~/.ltap or --type")
+end
+
+unless ARGV.first
+  abort(opts.help)
+end
+
+drainer = case conf.type
+when "papertrail"
+  Hutils::Ltap::PaperTrailDrainer
+when "splunk"
+  Hutils::Ltap::SplunkDrainer
+else
+  abort("Unknown type: #{conf.type}")
+end
+
+drainer = drainer.new(
+  key: conf.key,
+  query: ARGV.first,
+  timeout: conf.timeout,
+  url: conf.url,
+  verbose: conf.verbose
+)
+
+thread = Thread.start do
+  $stdout.puts drainer.run
+end
+
+# cancel a running job on a SIGINT because Splunk search slots are apparently a
+# very valuable limited resource
+trap('SIGINT', 'SIGTERM') do
+  drainer.cancel_job
+  thread.terminate
+  abort("Caught deadly signal")
+end
+
+thread.join

data/bin/lviz

@@ -0,0 +1,71 @@
+#!/usr/bin/env ruby
+
+require "inifile"
+require "optparse"
+require_relative "../lib/hutils"
+
+colors = $stdout.tty?
+compact = false
+highlights = %w(endpoint path route_signature user user_id)
+ignore = %w(at)
+interactive = false
+
+ini = IniFile.load(ENV["HOME"] + "/.lviz")
+if g = ini && ini["global"]
+  colors = g["colors"] if g["colors"]
+  compact = g["compact"] if g["compact"]
+  highlights = g["highlights"].split(",") if g["highlights"]
+  ignore = g["ignore"].split(",") if g["ignore"]
+end
+
+opts = OptionParser.new do |opts|
+  opts.banner = "Usage: lviz [options] <file> ..."
+  opts.on("--compact", "Compact display") { |c| compact = c }
+  opts.on("-h", "--help", "Show this help string") { |h|
+    if h
+      puts(opts.help)
+      exit(0)
+    end
+  }
+  opts.on("--highlights", "Keys to highlight (comma separated)") { |h|
+    highlights = h.split(",") rescue []
+  }
+  opts.on("--ignore", "Keys to ignore (comma separated)") { |i|
+    ignore = i.split(",") rescue []
+  }
+  opts.on("-i", "--interactive", "Interactive mode") { |i| interactive = i }
+  opts.on("--no-color", "Disable colors") { |c| colors = c }
+end
+opts.parse!
+
+# ARGF will read from stdin if available, and otherwise fall back to files
+# named as parameters (similar to the behavior of `cat`)
+lines = Hutils::Parser.new(ARGF.read).parse
+Hutils::Stripper.new(lines, ignore).run
+root = Hutils::TreeBuilder.new(lines).build
+
+if interactive
+  require_relative "../lib/hutils/curses_visualizer"
+  require_relative "../lib/hutils/node_navigator"
+
+  unless $stdin.tty?
+    abort("Can't start in interactive mode if STDIN isn't a TTY")
+  end
+
+  Hutils::CursesVisualizer.new(
+    colors: colors,
+    compact: compact,
+    highlights: highlights,
+    root: root
+  ).run
+else
+  require_relative "../lib/hutils/text_visualizer"
+
+  Hutils::TextVisualizer.new(
+    colors: colors,
+    compact: compact,
+    highlights: highlights,
+    out: $stdout,
+    root: root
+  ).display
+end

data/lib/hutils/curses_visualizer.rb

@@ -0,0 +1,207 @@
+require "curses"
+
+module Hutils
+  class CursesVisualizer
+    def initialize(colors:, compact:, highlights:, root:)
+      @colors = colors
+      @highlights = highlights
+      @line_buffer = []
+      @root = root
+    end
+
+    def run
+      Curses.cbreak      # no need for a newline to get type chars
+      Curses.curs_set(0) # invisible
+      Curses.noecho      # don't echo character on a getch
+
+      Curses.start_color
+      Curses.use_default_colors
+
+      Curses.init_pair(COLOR_KEY, -1, Curses::COLOR_BLUE)
+      Curses.init_pair(COLOR_HIGHLIGHT, Curses::COLOR_BLACK, Curses::COLOR_YELLOW)
+
+      Curses.init_screen
+
+      trap("SIGINT", "SIGTERM") do
+        Curses.close_screen
+        # @todo: curses shutdown?
+        $stdout.puts "Caught deadly signal"
+        $stdout.flush
+        exit(0)
+      end
+
+      # set root node as "expanded"
+      @root.tags[:expanded] = true
+
+      @need_repaint = true
+      @selected_line = 0
+      build_line_buffer
+
+      loop do
+        if @need_repaint
+          build_line_buffer
+          paint
+          @need_repaint = false
+        end
+        handle_key
+      end
+    end
+
+    private
+
+    COLOR_KEY = 1
+    COLOR_HIGHLIGHT = 2
+
+    def build_line_buffer
+      @line_buffer = []
+
+      traverse_tree(@root) do |node|
+        @line_buffer << node
+      end
+
+      # remove the root node; it's not actually displayed
+      @line_buffer.shift
+
+      if @selected_line >= @line_buffer.count
+        @selected_line = @line_buffer.count - 1
+      end
+    end
+
+    def traverse_tree(node, &block)
+      yield node
+      if node.tags[:expanded]
+        node.slots.each { |child| traverse_tree(child, &block) }
+      end
+    end
+
+    def collapse_all
+      collapse_node(@root)
+    end
+
+    def collapse_node(node)
+      node.tags[:expanded] = false if node != @root
+      node.slots.each { |child| collapse_node(child) }
+    end
+
+    def expand_all(node = @root)
+      node.tags[:expanded] = true if node != @root
+      node.slots.each { |child| expand_all(child) }
+    end
+
+    def toggle_node
+      if node = @line_buffer[@selected_line]
+        node.tags[:expanded] = !node.tags[:expanded]
+      end
+    end
+
+    def expanded_str(node)
+      if node.slots.count > 0
+        node.tags[:expanded] ? "[-]" : "[+]"
+      else
+        "   "
+      end
+    end
+
+    def handle_key
+      case Curses.getch
+      when Curses::KEY_RESIZE then need_repaint
+      when ' ' then toggle_node && need_repaint
+      when ?c then collapse_all && need_repaint
+      when ?e then expand_all && need_repaint
+      when ?j then move_next
+      when ?k then move_prev
+      when ?o then toggle_node && need_repaint
+      when ?q then exit(0)
+      end
+    end
+
+    def move_next
+      old_selected_line = @selected_line
+
+      @selected_line += 1
+      if @selected_line == @line_buffer.count
+        @selected_line = @line_buffer.count - 1
+      end
+
+      paint_line(old_selected_line)
+      paint_line(@selected_line)
+    end
+
+    def move_prev
+      old_selected_line = @selected_line
+
+      @selected_line -= 1
+      if @selected_line < 0
+        @selected_line = 0
+      end
+
+      paint_line(old_selected_line)
+      paint_line(@selected_line)
+    end
+
+    def need_repaint
+      @need_repaint = true
+    end
+
+    def paint
+      Curses.clear
+
+      @line_buffer.each_with_index do |node, line|
+        break if line > Curses.lines
+        paint_node(node, line)
+      end
+
+      Curses.refresh
+    end
+
+    def paint_line(line)
+      paint_node(@line_buffer[line], line)
+    end
+
+    def safe_addstr(str, num_cols_written)
+      if num_cols_written + str.length > Curses.cols
+        left = Curses.cols - num_cols_written
+        str = str.dup[0, left]
+      end
+      Curses.addstr(str)
+      num_cols_written + str.length
+    end
+
+    # `line` should already be incremented to the correct position before
+    # entering this method
+    def paint_node(node, line)
+      n = 0
+      Curses.setpos(line, 0)
+
+      if @selected_line == line
+        Curses.attron(Curses::A_UNDERLINE)
+      end
+
+      n = safe_addstr("\t" * (node.depth - 1), n)
+      n = safe_addstr(expanded_str(node) + " ", n)
+      node.common.to_a.sort_by { |k, v| k }.each do |k, v|
+        if v == true
+          color(COLOR_KEY) { n = safe_addstr("#{k}", n) }
+        else
+          if @highlights.include?(k)
+            color(COLOR_HIGHLIGHT) { n = safe_addstr("#{k}=#{v}", n) }
+          else
+            color(COLOR_KEY) { n = safe_addstr("#{k}", n) }
+            n = safe_addstr("=#{v}", n)
+          end
+        end
+        n = safe_addstr(" ", n)
+      end
+
+      Curses.attroff(Curses::A_UNDERLINE)
+    end
+
+    def color(key, &block)
+      if @colors
+        Curses.attron(Curses::color_pair(key) | Curses::A_NORMAL) { yield }
+      else
+        yield
+      end
+    end
+  end
+end

data/lib/hutils/ltap/conf.rb

@@ -0,0 +1,41 @@
+require "inifile"
+
+module Hutils::Ltap
+  class Conf
+    attr_accessor :key
+    attr_accessor :profile
+    attr_accessor :timeout
+    attr_accessor :type
+    attr_accessor :url
+    attr_accessor :verbose
+
+    def initialize
+      @ini = IniFile.load(ENV["HOME"] + "/.ltap")
+      self.timeout = 60
+      self.verbose = false
+    end
+
+    def load
+      load_section("global")
+    end
+
+    def load_section(name)
+      if section = @ini && @ini[name]
+        load_value(section, :key)
+        load_value(section, :profile)
+        load_value(section, :timeout)
+        load_value(section, :type)
+        load_value(section, :url)
+        load_value(section, :verbose)
+      end
+    end
+
+    private
+
+    def load_value(section, name)
+      if value = section[name.to_s]
+        send("#{name}=", value)
+      end
+    end
+  end
+end

data/lib/hutils/ltap/paper_trail_drainer.rb

@@ -0,0 +1,75 @@
+require "json"
+
+module Hutils::Ltap
+  class PaperTrailDrainer
+    PAPER_TRAIL_URL = "https://papertrailapp.com"
+
+    def initialize(key:, timeout:, query:, url:, verbose:)
+      @api = Excon.new(PAPER_TRAIL_URL,
+        headers: {
+          "X-Papertrail-Token" => key
+        })
+      @query = query
+      @timeout = timeout
+      @verbose = verbose
+    end
+
+    def run
+      messages = []
+      min_id = nil
+      start = Time.now
+
+      loop do
+        new_messages, reached_beginning, min_id = fetch_page(min_id)
+        messages += new_messages
+
+        # break if PaperTrail has indicated that we've reached the beginning of
+        # our results, or if we've approximately hit our timeout
+        if reached_beginning || (Time.now - start).to_i > @timeout
+          break
+        end
+      end
+
+      messages
+    rescue RateLimited
+      $stderr.puts "Papertrail rate limit reached"
+      messages
+    end
+
+    def cancel_job
+      debug("cancelled [noop]")
+    end
+
+    private
+
+    class RateLimited < StandardError
+    end
+
+    def debug(str)
+      if @verbose
+        puts str
+      end
+    end
+
+    def fetch_page(max_id)
+      resp = @api.get(
+        path: "/api/v1/events/search.json",
+        expects: [200, 429],
+        query: {
+          max_id: max_id,
+          q: @query
+        }.reject { |k, v| v == nil })
+
+      if resp.status == 429
+        raise RateLimited
+      end
+
+      data = JSON.parse(resp.body)
+      debug("backend_timeout: #{data["backend_timeout"] || false} " +
+        "min_id: #{data["min_id"]} " +
+        "reached_beginning: #{data["reached_beginning"] || false}")
+      messages = data["events"].map { |e| e["message"].strip }
+      [messages, data["reached_beginning"], data["min_id"]]
+    end
+  end
+end

data/lib/hutils/ltap/splunk_drainer.rb

@@ -0,0 +1,123 @@
+require "csv"
+require "excon"
+require "json"
+require "uri"
+
+module Hutils::Ltap
+  class SplunkDrainer
+    def initialize(key:, timeout:, query:, url:, verbose:)
+      @timeout = timeout
+      @query = query
+      @verbose = verbose
+
+      @user = URI.parse(url).user
+      @api = Excon.new(url)
+    end
+
+    def run
+      create_job(@query)
+      start = Time.now
+
+      loop do
+        sleep(2)
+        break if job_finished?
+
+        # finalize the job if we've broken our timeout point
+        if (Time.now - start).to_i > @timeout
+          finalize_job
+          break
+        end
+      end
+
+      get_job_results
+    end
+
+    def cancel_job
+      return unless @job_id
+      @api.post(
+        path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}/control",
+        expects: 200,
+        body: URI.encode_www_form({
+          action: "cancel"
+        })
+      )
+      debug("cancelled")
+    end
+
+    private
+
+    def create_job(query)
+      resp = @api.post(
+        path: "/servicesNS/#{@user}/search/search/jobs",
+        expects: 201,
+        body: URI.encode_www_form({
+          output_mode: "json",
+          search: "search #{query}"
+        })
+      )
+      @job_id = JSON.parse(resp.body)["sid"]
+      debug "job: #{@job_id}"
+    end
+
+    def debug(str)
+      if @verbose
+        puts str
+      end
+    end
+
+    def finalize_job
+      @api.post(
+        path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}/control",
+        expects: 200,
+        body: URI.encode_www_form({
+          action: "finalize"
+        })
+      )
+      debug("finalized")
+    end
+
+    def get_job_results
+      # get results as CSV because the JSON version just mixes everything together
+      # into a giant difficult-to-use blob
+      resp = @api.get(
+        path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}/results",
+        # 204 if no results available
+        expects: [200, 204],
+        body: URI.encode_www_form({
+          action: "finalize",
+          output_mode: "csv"
+        })
+      )
+
+      return [] if resp.status == 204
+
+      rows = CSV.parse(resp.body)
+      return [] if rows.count < 1
+      field = rows[0].index("_raw") || raise("no _raw field detected in Splunk response")
+
+      # skip the first line as its used for CSV headers
+      rows[1..-1].
+        map { |l| l[field] }.
+        # 2014-08-15T19:01:15.476590+00:00 54.197.117.24 local0.notice
+        # api-web-1[23399]: - api.108080@heroku.com ...
+        map { |l| l.gsub(/^.*: - /, "") }.
+        map { |l| l.strip }.
+        # results come in from newest to oldest; flip that
+        reverse
+    end
+
+    def job_finished?
+      resp = @api.get(
+        path: "/servicesNS/#{@user}/search/search/jobs/#{@job_id}",
+        expects: 200,
+        body: URI.encode_www_form({
+          output_mode: "json"
+        })
+      )
+      # Splunk may not be winning any awards for cleanest API anytime soon
+      state = JSON.parse(resp.body)["entry"][0]["content"]["dispatchState"]
+      debug("state: #{state}")
+      state == "DONE"
+    end
+  end
+end

data/lib/hutils/ltap.rb

@@ -0,0 +1,3 @@
+require_relative "ltap/conf"
+require_relative "ltap/paper_trail_drainer"
+require_relative "ltap/splunk_drainer"

data/lib/hutils/node_navigator.rb

@@ -0,0 +1,52 @@
+module Hutils
+  class NodeNavigator
+    def next_node(node, ignore_expanded: false)
+      # if expanded and we have children, move onto the first
+      if !ignore_expanded && node.tags[:expanded] && node.slots.count > 0
+        return node.slots[0]
+      end
+
+      index = node.parent.slots.index(node)
+
+      # Otherwise, if the node is the last in its slot, then move to the
+      # parent's next slot. If the parent is root, then we can go no further.
+      if index == node.parent.slots.count - 1
+        if node.parent.parent != nil
+          new_node = next_node(node.parent, ignore_expanded: true)
+          # if the sub-iteration couldn't find a next node, stay where we are
+          new_node != node.parent ? new_node : node
+        else
+          node
+        end
+      # otherwise, just move to the next slot
+      else
+        node.parent.slots[index + 1]
+      end
+    end
+
+    def prev_node(node)
+      index = node.parent.slots.index(node)
+
+      if index == 0
+        if node.parent.parent != nil
+          node.parent
+        else
+          # don't ever move up to root
+          return node
+        end
+      # otherwise, move to the previous node in the list
+      else
+        new_node = node.parent.slots[index - 1]
+
+        # But wait! We don't just want to move to the previous node directly, we
+        # actually want to move to the last child of its deepest expanded
+        # subnode.
+        while new_node.tags[:expanded] && new_node.slots.count > 0
+          new_node = new_node.slots.last
+        end
+
+        new_node
+      end
+    end
+  end
+end

data/lib/hutils/stripper.rb

@@ -0,0 +1,14 @@
+module Hutils
+  class Stripper
+    def initialize(messages, ignore)
+      @messages = messages
+      @ignore = ignore
+    end
+
+    def run
+      @messages.each do |message|
+        message.reject! { |k, _| @ignore.include?(k) }
+      end
+    end
+  end
+end

data/lib/hutils/text_visualizer.rb

@@ -0,0 +1,61 @@
+require "term/ansicolor"
+
+module Hutils
+  class TextVisualizer
+    include Term::ANSIColor
+
+    def initialize(colors:, compact:, highlights:, root:, out:)
+      @colors = colors
+      @compact = compact
+      @highlights = highlights
+      @out = out
+      @root = root
+    end
+
+    def display
+      display_node(@root)
+    end
+
+    private 
+
+    def colorize(method, str)
+      if @colors
+        send(method, str)
+      else
+        str
+      end
+    end
+
+    def display_node(node)
+      if !node.common.empty?
+        # the "- 1" is because the root node is empty
+        indent = "\t" * (node.depth - 1)
+        node.common.to_a.sort_by { |k, v| k }.map { |k, v|
+          pair_to_string(k, v)
+        }.each_with_index { |display, i|
+          if @compact
+            @out.print(indent) if i == 0
+            @out.print("#{display} ")
+          else
+            marker = i == 0 ? "+ " : "  "
+            @out.puts "#{indent}#{marker}#{display}"
+          end
+        }
+        @out.puts ""
+      end
+      node.slots.each { |slot| display_node(slot) }
+    end
+
+    def pair_to_string(k, v)
+      if v == true
+        colorize(:green, k)
+      else
+        if @highlights.include?(k)
+          colorize(:on_yellow, colorize(:black, "#{k}: #{v}"))
+        else
+          "#{colorize(:green, k)}: #{v}"
+        end
+      end
+    end
+  end
+end

data/lib/hutils.rb

@@ -0,0 +1,177 @@
+require_relative "hutils/stripper"
+
+module Hutils
+  class Node
+    def initialize(parent, common)
+      @common = common
+      @parent = parent
+      @slots = []
+      @tags = {}
+    end
+
+    # The set of common attributes that are shared by all slots in this branch of
+    # the tree.
+    attr_accessor :common
+
+    # Parent node.
+    attr_accessor :parent
+
+    # Ordered child nodes of this node.
+    attr_accessor :slots
+
+    # Arbitrary tags that applications can associate on a node.
+    attr_accessor :tags
+
+    # The set of common attributes that are shared by all slots in this branch of
+    # the tree, but accounting for all parent nodes as well.
+    def common_complete
+      @common.merge(@parent ? @parent.common_complete : {})
+    end
+
+    def depth
+      @parent ? @parent.depth + 1 : 0
+    end
+
+    def print
+      indent = "  " * depth
+      $stdout.puts "#{indent}[ #{depth} ]#{@common}"
+      slots.each { |node| node.print }
+    end
+
+    def replace_slot(old, new)
+      @slots.each_with_index do |node, i|
+        if old == node
+          @slots[i] = new
+          return
+        end
+      end
+      abort("bad replace")
+    end
+
+    def root
+      @parent ? @parent.root : self
+    end
+
+    def root?
+      @parent == nil
+    end
+  end
+
+  class Parser
+    def initialize(str)
+      @str = str
+    end
+
+    def parse
+      lines = @str.split("\n").map { |line| normalize(line) }
+      lines.map! do |line|
+        pairs = line.scan(/(?:['"](?:\\.|[^'"])*['"]|[^'" ])+/).map do |pair|
+          key, value = pair.split("=")
+          [key, value].each do |str|
+            str.gsub!(/^['"]?(.*?)['"]?$/, '\1') if str
+          end
+          [key, value || true]
+        end
+        Hash[pairs]
+      end
+    end
+
+    private
+
+    def normalize(line)
+      line = line.strip
+      line.gsub(/^[T0-9\-:+.]+( [a-z]+\[[a-z0-9\-_.]+\])?: /, '')
+    end
+  end
+
+  class TreeBuilder
+    def initialize(lines)
+      @lines = lines
+    end
+
+    def build
+      root = Node.new(nil, {})
+      node = root
+      @lines.each do |pairs|
+        node = build_node(node, pairs)
+      end
+      root
+    end
+
+    private
+
+    def diff(hash1, hash2)
+      same = hash1.dup.delete_if { |k, v| hash2[k] != v }
+
+      extra1 = hash1.dup
+      same.each { |k, _| extra1.delete(k) }
+
+      extra2 = hash2.dup
+      same.each { |k, _| extra2.delete(k) }
+
+      [same, extra1, extra2]
+    end
+
+    def debug(title, data, node)
+      return unless ENV["DEBUG"] == "true"
+      puts "---"
+      puts title
+      data.each { |k, v| puts "#{k}: #{v}" }
+      node.root.print if node
+      puts ""
+    end
+
+    def build_node(node, pairs)
+      complete = node.common_complete
+      same, complete_extra, pairs_extra = diff(complete, pairs)
+
+      if complete == same
+        if pairs_extra.empty?
+          # we've hit a rare case of a line which is a duplicate of its immediate
+          # successor: do nothing
+        else
+          # all common is shared but we have extra: add a new child node
+          new = Node.new(node, pairs_extra)
+          node.slots << new
+          node = new
+          debug("simple leaf addition", { pairs_extra: pairs_extra }, node)
+        end
+      else
+        # First of all, determine whether our node has any data in common with
+        # the current node.
+        local_same, _, _ = diff(node.common, pairs)
+
+        # Then figure out that given a split, what the old node's new set would
+        # look like in the context of a shared common parent.
+        _, other_extra, _ = diff(node.common, local_same)
+
+        # And finally, determine whether the parent hierarchy contains any data
+        # that is incompatible with our new node because it's not shared.
+        _, illegal_extra, _ = diff(complete_extra, other_extra)
+
+        if !local_same.empty? && illegal_extra.empty?
+          # create a replacement node and swap it into place
+          new_parent = Node.new(node.parent, local_same)
+          node.common = other_extra
+          node.parent = new_parent
+          new_parent.slots << node
+          new_parent.parent.replace_slot(node, new_parent)
+
+          new = Node.new(new_parent, pairs_extra)
+          new_parent.slots << new
+          node = new
+          debug("split", { pairs_extra: pairs_extra }, node)
+        else
+          debug("before tree ascention", { pairs: pairs }, nil)
+          # Nothing is shared, ascend the tree until something is. Eventually
+          # something will be because the root node contains an empty hash of common
+          # attributes.
+          node = build_node(node.parent, pairs)
+          debug("after tree ascention", { pairs: pairs }, nil)
+        end
+      end
+
+      node
+    end
+  end
+end

metadata

@@ -0,0 +1,120 @@
+--- !ruby/object:Gem::Specification
+name: hutils
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Brandur
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-08-22 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: excon
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.39'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.39.5
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0.39'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.39.5
+- !ruby/object:Gem::Dependency
+  name: inifile
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '3.0'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  name: term-ansicolor
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.3'
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.3.0
+description: 
+email: brandur@mutelight.org
+executables:
+- lcut
+- ltap
+- lviz
+extensions: []
+extra_rdoc_files: []
+files:
+- "./lib/hutils.rb"
+- "./lib/hutils/curses_visualizer.rb"
+- "./lib/hutils/ltap.rb"
+- "./lib/hutils/ltap/conf.rb"
+- "./lib/hutils/ltap/paper_trail_drainer.rb"
+- "./lib/hutils/ltap/splunk_drainer.rb"
+- "./lib/hutils/node_navigator.rb"
+- "./lib/hutils/stripper.rb"
+- "./lib/hutils/text_visualizer.rb"
+- README.md
+- bin/lcut
+- bin/ltap
+- bin/lviz
+homepage: https://github.com/brandur/hutils
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: A collection of command line utilities for working with logfmt.
+test_files: []
+has_rdoc: