sawmill-sawyer 13.0.2

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 55137b531ea6620f6a0f27a0fe51569e8240118f
+  data.tar.gz: 1bc7ceaf34cd708a76dc75974d0e77a4e0df3443
+SHA512:
+  metadata.gz: bd78951fbe1e60bacf54c7ef8a4419a587bdcb9aca0b0e57ef9366f40b3eaf3104f44bf81ef38bf8fdb5064555fa72bb8a2fe5bd13b2a962a8739c939b93223e
+  data.tar.gz: 4234a4d8e3c6a1f26b346f28e161de5cc9801fe9ea6e23071eb4c9fb9bcb652e405dc26c6fde05b0d747dc9b94e812b9cbfef41546e33fb2f47547b5b331ce31

data/.gitignore

@@ -0,0 +1,9 @@
+*~
+*.log
+*.gem
+*.out
+*.pid
+*.swp
+*.lock
+tmp
+.DS_Store

data/Gemfile

@@ -0,0 +1,17 @@
+source 'https://rubygems.org'
+
+gemspec
+
+group :development do
+  gem 'pry'
+  gem 'rake'
+  gem 'yard'
+  gem 'version'
+  gem 'rubygems-tasks'
+end
+
+group :test do
+  gem 'minitest'
+  gem 'simplecov'
+  gem 'rubocop'
+end

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2014 Sean Clemmer and Blue Jeans Network
+
+Permission to use, copy, modify, and/or distribute this software for any
+purpose with or without fee is hereby granted, provided that the above
+copyright notice and this permission notice appear in all copies.
+
+THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH
+REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY
+AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT,
+INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM
+LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR
+OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THIS SOFTWARE.

data/Rakefile

@@ -0,0 +1,29 @@
+require 'rubygems'
+require 'bundler'
+require 'rake'
+
+require 'rake/testtask'
+Rake::TestTask.new(:test) do |test|
+  test.libs << 'lib' << 'test'
+  test.test_files = FileList['test/test*.rb']
+  test.verbose = true
+end
+
+task default: :test
+
+
+require 'yard'
+YARD::Rake::YardocTask.new do |t|
+  t.files = %w[ --readme Readme.md lib/**/*.rb - VERSION ]
+end
+
+
+require 'rubygems/tasks'
+Gem::Tasks.new push: true, sign: {} do |tasks|
+  tasks.console.command = 'pry'
+end
+Gem::Tasks::Sign::Checksum.new sha2: true
+
+
+require 'rake/version_task'
+Rake::VersionTask.new

data/Readme.md

@@ -0,0 +1,3 @@
+# Sawyer
+
+See [the Sawyer page](http://wiki.bluejeansnet.com/operations/sawmill/sawyer) on the Blue Jeans Wiki for documentation.

data/VERSION

@@ -0,0 +1 @@
+13.0.2

data/bin/sawyer

@@ -0,0 +1,56 @@
+#!/usr/bin/env ruby
+require_relative '../lib/sawyer'
+
+BIN_DIR = File.dirname(File.expand_path(__FILE__))
+VERSION = File.read(File.join(BIN_DIR, '..', 'VERSION')).strip
+ABOUT   = "sawyer v#{VERSION} - Sean Clemmer (sclemmer@bluejeans.com) / Blue Jeans Network"
+ACTIONS = [
+  "logs QUERY_STRING  --  Find all logs matching the query string within an interval",
+  "events QUERY_STRING [LOGS] --  Find all events matching the query string within the given logs and interval"
+]
+
+argv = ARGV.empty? ? %w[ --help ] : ARGV
+
+opts = Trollop::options(argv) do
+  version ABOUT
+
+  ART = <<-'EOART' % VERSION
+                                      .---.                      __  ,-.
+        .--.--.                      /. ./|                    ,' ,'/ /|
+       /  /    '    ,--.--.       .-'-. ' |      .--,   ,---.  '  | |' |
+      |  :  /`./   /       \     /___/ \: |    /_ ./|  /     \ |  |   ,'
+      |  :  ;_    .--.  .-. | .-'.. '   ' . , ' , ' : /    /  |'  :  /
+       \  \    `.  \__\/: . ./___/ \:     '/___/ \: |.    ' / ||  | '
+        `----.   \ ," .--.; |.   \  ' .\    .  \  ' |'   ;   /|;  : |
+       /  /`--'  //  /  ,.  | \   \   ' \ |  \  ;   :'   |  / ||  , ;
+      '--'.     /;  :   .'   \ \   \  |--"    \  \  ;|   :    | ---'
+        `--'---' |  ,     .-./  \   \ |        :  \  \\   \  /
+                  `--`---'       '---"          \  ' ; `----'
+                                                 `--`             v%s
+  EOART
+
+  USAGE = <<-EOS.gsub(/^    /, '')
+    Usage: sawyer [OPTIONS] ACTION [ARGS]
+
+    Actions:
+    \t#{ACTIONS.join("\n\t")}
+
+    Options:
+  EOS
+
+  banner "\n\n" + ART + "\n" + USAGE
+  opt :from, 'Match events after this', default: '15 minutes ago'
+  opt :to, 'Match events before this',  default: 'now'
+  opt :environment, 'Match events in this environment', default: 'prod'
+  opt :prefix, 'Match events with this index prefix', default: 'logs'
+  opt :limit, 'Limit search result size',  default: 25
+  opt :debug, 'Enable debug output',       default: false
+end
+
+level  = ::Logger::INFO
+level  = ::Logger::DEBUG if opts[:debug]
+logger = Sawyer::Logger.new STDERR, level
+action = argv.shift.to_sym rescue nil
+
+Trollop::die 'No action provided' if action.nil?
+exit Sawyer::Actions.new(opts, action, argv, logger).status

data/lib/sawyer.rb

@@ -0,0 +1,13 @@
+require 'json'
+require 'date'
+require 'time'
+require 'logger'
+require 'pathname'
+require 'fileutils'
+
+require 'curb'
+require 'chronic'
+require 'trollop'
+
+require_relative 'sawyer/logger'
+require_relative 'sawyer/actions'

data/lib/sawyer/actions.rb

@@ -0,0 +1,99 @@
+require_relative 'actions_raw'
+
+class Fixnum
+  def hours
+    self * 60 * 60 # [hr] * [min/hr] * [s/min] => [s]
+  end
+end
+
+
+module Sawyer
+  ENVIRONMENTS = {
+    prod: 'http://10.100.0.146:8080/es',
+    stage: 'http://10.100.200.12:8080/es'
+  }
+
+  class Actions
+    include RawActions
+
+    attr_reader :logger, :status
+
+    def initialize opts, action, args, logger
+      environment = opts[:environment].to_sym
+      from        = Chronic.parse(opts[:from])
+      to          = Chronic.parse(opts[:to])
+
+      @prefix = opts[:prefix]
+      @limit  = opts[:limit]
+      @action = action
+      @args   = args
+      @logger = logger
+      @status = 0
+
+      @elasticsearch = ENVIRONMENTS[environment.to_sym]
+
+      @status = send action, from, to, args
+    end
+
+  private
+    def search from, to, request
+      indexes   = []
+
+      # INF-4114: Limit searches to 24h max
+      max   = 24
+      limit = from + max.hours
+      if to > limit
+        @logger.warn "Truncating search to first #{max} hours"
+        to = limit
+      end
+
+      date_from = from.getutc.to_date
+      date_to   = to.getutc.to_date
+
+      until date_from > date_to
+        indexes << date_from.strftime("#{@prefix}-%Y.%m.%d")
+        date_from += 1 # day
+      end
+
+      unless request.has_key?(:size) and request[:size] == 0
+        request[:size] = @limit
+      end
+
+      if request.has_key? :aggregations
+        request[:aggregations].each do |name, agg|
+          agg.keys.each do |k|
+            request[:aggregations][name][k].merge!({ size: @limit })
+          end
+        end
+      end
+
+      request[:query] = {
+        filtered: {
+          query: request[:query],
+          filter: {
+            bool: {
+              must: {
+                range: {
+                  '@timestamp' => {
+                    gte: from.iso8601,
+                    lte: to.iso8601
+                  }
+                }
+              }
+            }
+          }
+        }
+      }
+
+      url = "#{@elasticsearch}/#{indexes * ','}/_search?ignore_unavailable=true&timeout=10000"
+      @logger.debug url
+      @logger.debug request.to_json
+      curl_request = Curl::Easy.http_post(url, request.to_json) do |curl|
+        curl.headers['Accept'] = 'application/json'
+        curl.headers['Content-Type'] = 'application/json'
+        curl.verbose = @logger.debug?
+      end
+      JSON::parse curl_request.body_str
+    end
+  end
+end

data/lib/sawyer/actions_raw.rb

@@ -0,0 +1,69 @@
+module RawActions
+  def logs from, to, args
+    qs = args.shift
+    request = {
+      size: 0,
+      query: {
+        query_string: {
+          query: qs
+        }
+      },
+      aggregations: {
+        logs: {
+          terms: {
+            script: "_source.host + _source.path"
+          }
+        }
+      }
+    }
+
+    response = search from, to, request
+    buckets = []
+    if response.has_key? 'aggregations'
+      buckets = response['aggregations']['logs']['buckets']
+    end
+    logs = buckets.map { |log| "%d\t%s" % [ log['doc_count'], log['key'] ] }
+    puts logs.reverse
+    return 0
+  end
+
+
+
+  def events from, to, args
+    qs   = args.shift
+    logs = args
+
+    $stdin.read.split("\n").each do |l|
+      logs << l.split(' ', 2).last
+    end unless $stdin.tty?
+
+    unless logs.empty?
+      logs = logs.map do |log|
+        host, path = log.split('/', 2)
+        if path.nil? || path.empty?
+          'host:"%s"' % host
+        else
+          '(host:"%s" AND path:"/%s")' % [ host, path ]
+        end
+      end.join(' OR ')
+      logs = " AND (#{logs})"
+    else
+      logs = ''
+    end
+
+    request = {
+      query: {
+        query_string: {
+          query: qs + logs
+        }
+      }
+    }
+
+    response = search from, to, request
+    if response.has_key?('hits')
+      puts JSON::pretty_generate(response['hits']['hits'].map { |e| e['_source'] })
+      return 0
+    end
+    return 1
+  end
+end

data/lib/sawyer/logger.rb

@@ -0,0 +1,12 @@
+module Sawyer
+  module Logger
+    def self.new file=STDOUT, level=::Logger::INFO
+      logger           = ::Logger.new file
+      logger.level     = level
+      logger.formatter = proc { |severity, datetime, _, msg|
+        "#{severity} [#{datetime.iso8601(6)}] #{File::basename caller[4]} -- #{msg}\n"
+      }
+      return logger
+    end
+  end
+end

data/sawmill-sawyer.gemspec

@@ -0,0 +1,20 @@
+Gem::Specification.new do |s|
+  s.name        = 'sawmill-sawyer'
+  s.version     = File.read('VERSION').strip
+  s.platform    = Gem::Platform::RUBY
+  s.author      = 'Sean Clemmer'
+  s.email       = 'sclemmer@bluejeans.com'
+  s.homepage    = 'http://wiki.bluejeansnet.com/operations/sawyer'
+  s.summary     = 'Sawyer is a command-line companion to Sawmill'
+  s.description = 'Sawyer is a command-line companion to Sawmill.'
+  s.license     = 'ISC'
+
+  s.add_runtime_dependency 'curb', '~> 0'
+  s.add_runtime_dependency 'chronic', '~> 0'
+  s.add_runtime_dependency 'trollop', '~> 2'
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ['lib']
+end

metadata

@@ -0,0 +1,98 @@
+--- !ruby/object:Gem::Specification
+name: sawmill-sawyer
+version: !ruby/object:Gem::Version
+  version: 13.0.2
+platform: ruby
+authors:
+- Sean Clemmer
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2014-12-25 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: curb
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: chronic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: trollop
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '2'
+description: Sawyer is a command-line companion to Sawmill.
+email: sclemmer@bluejeans.com
+executables:
+- sawyer
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- LICENSE
+- Rakefile
+- Readme.md
+- VERSION
+- bin/sawyer
+- lib/sawyer.rb
+- lib/sawyer/actions.rb
+- lib/sawyer/actions_raw.rb
+- lib/sawyer/logger.rb
+- sawmill-sawyer.gemspec
+homepage: http://wiki.bluejeansnet.com/operations/sawyer
+licenses:
+- ISC
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.2.2
+signing_key: 
+specification_version: 4
+summary: Sawyer is a command-line companion to Sawmill
+test_files: []