sumo-search 0.1.1 → 1.0.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 193a26c19dd5ba23174ee1bbb198c691d7374744
-  data.tar.gz: 52a5b12148a20750ac5a1d3213c082560e8fbfce
+  metadata.gz: 62af7dc96394b1bea0a7bff6a0f92e0b7a8c9ba8
+  data.tar.gz: e22438c39474302d64d49765e4833c7be5821eec
 SHA512:
-  metadata.gz: 302b0a034afc2813476b992ce98fb5fd74d85b9094945080738d82c4150e208ea14c05dcaf1a9da73bf9315be4cbafbce7ad2faf47216ae43d39a6377b2dfb54
-  data.tar.gz: f82388e8f5adbde5107e54e6ba7a0e8938586bcd882b60249e719808d56b67fde7989ae6dcfc19adafb510315b6ed202166d6396abc19227e3dfe1d78a1b0700
+  metadata.gz: 8cb8205eb109d953c1764cc293f4134776b882630747ca2e9ef5b46dc326a17c5e89f46ffec02ca36e1ba1d59215e66c57f81fc89ac7b41c99b5865ca10022ab
+  data.tar.gz: 52ac1a59652216c58d6c873333605589c5621a6e221f6ccb91424fdd62a6614650b7b37873d178c36ccfc6e1fc21990999eee29d8ea5447607b0f5e3b2ad8900

data/.gitignore

@@ -2,5 +2,3 @@
 *.swp
 *.gem
 Gemfile.lock
-coverage/
-.rbenv-version

data/.travis.yml

@@ -0,0 +1,7 @@
+language: ruby
+rvm:
+    - 1.9.2
+    - 1.9.3
+    - 2.0.0
+    - 2.1.0
+script: bundle exec rake --trace

data/README.md

@@ -1,45 +1,72 @@
 # sumo-search
 
-This library can be used to query Sumo Logic's search API.
+This gem interfaces with the Sumo Logic [Search Job API](https://github.com/SumoLogic/sumo-api-doc/wiki/search-job-api).
+It may be used through native Ruby, or via a CLI that has been provided.
 
 ## Installation
 
 From the command line:
 
-```shell
-$ sudo gem install sumo-search
+```bash
+$ [sudo] gem install sumo-search
 ```
 
-Installing this gem will install a script called `sumo`.
-Note that this gem requires a Ruby version of at least 2.0.0.
+From your application's `Gemfile`:
 
-## Configuration
+```ruby
+gem 'sumo-search'
+```
+
+## Ruby Usage
 
-To use this script, you must supply your Sumo Logic credentials.
-The script can read credentials from one of two places; either an environment variable or a configuration file.
-The environment variable is `SUMO_CREDS`, and the configuration file is `~/.sumo_creds`.
-Whichever you choose, the credentials must be in the following format:
+To create a search job from ruby, the `Sumo.search` method is provided.
+For example, the following creates a search job for everything from the 2014-01-01:
 
+```ruby
+search = Sumo.search(
+  :query => '*',
+  :from => '2014-01-01T00:00:00',
+  :to => '2014-01-01T23:59:59',
+  :time_zone => 'UTC'
+)
 ```
-username@website:password
+
+To iterate through the messages returned by the API, use the `#messages` method on the object returned by `Sumo.search`.
+
+```ruby
+search.messages.each { |message| puts message }
 ```
 
-## Usage
+Similarly, iterating through the records can be acheived through the `#records` method.
 
-```shell
-$ sumo [*options/flags*]
+```ruby
+search.records.each { |record| puts record }
 ```
 
-This script wraps the [Sumo Search API](https://github.com/SumoLogic/sumo-api-doc/wiki/search-api).
-The API only has one endpoint, `/api/v1/logs/search` which searches your logs.
-
-| Command Line Option    | Corresponding Query Parameter | Description                          |
-|------------------------|-------------------------------|--------------------------------------|
-| -q, --query QUERY      | q                             | The query to search (required)       |
-| -f, --from START       | from                          | Begin searching at START (ISO8601)   |
-| -t, --to END           | to                            | Finish searching at END (ISO8601)    |
-| -z, --time-zone ZONE   | tz                            | The timezone of the search           |
-| -c, --config-file FILE | -                             | Use the specified config file        |
-| -e, --extract-key KEY  | -                             | Extract the given key from your logs |
-| -h, --help             | -                             | Display the help message             |
-| -v, --version          | -                             | Display the version                  |
+Note that the two above methods lazily grab the results in chunks, so iterating through these will take some time.
+The difference between records and messages is described at the bottom of [this section](https://github.com/SumoLogic/sumo-api-doc/wiki/search-job-api#wiki-getting-the-current-search-job-status) of the api docs.
+
+## CLI Usage
+
+The executable packaged with this gem is called `sumo`.
+
+| Option           | Required | Description                                     |
+|------------------|----------|-------------------------------------------------|
+| -q --query       | `true`   | The query to send to the API                    |
+| -f --from        | `true`   | The start date of the query (iso8601)           |
+| -t --to          | `true`   | The end date of the query (iso8601)             |
+| -z --time-zone   | `true`   | The time zone of the start and end dates        |
+| -e --extract-key | `false`  | Extract the given key from the returned message |
+| -r --records     | `false`  | Print out the records, not messages             |
+| -v --version     | `false`  | Print the version and exit                      |
+| -h --help        | `false`  | Print the help message and exit.                |
+
+Examples:
+
+```bash
+# Search for everything from 2014-01-01.
+sumo --query '*' --from '2014-01-01T00:00:00' --to '2014-01-01T23:59:59' --time-zone 'UTC'
+
+# Search for everything containing 'StagingFitness' in 2013, extracting the 'message' key from the response.
+sumo --query 'StagingFitness' --from '2013-01-01T00:00:00' --to '2014-01-01T00:00:00' --time-zone 'UTC' --extract-key 'message'
+```

data/Rakefile

@@ -1,4 +1,4 @@
-$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), 'lib'))
+$LOAD_PATH << File.join(File.dirname(__FILE__), 'lib')
 
 require 'rake'
 require 'sumo'
@@ -7,5 +7,10 @@ require 'cane/rake_task'
 
 task :default => [:spec, :quality]
 
-RSpec::Core::RakeTask.new(:spec) { |t| t.pattern = 'spec/**/*_spec.rb' }
-Cane::RakeTask.new(:quality) { |t| t.canefile = '.cane' }
+RSpec::Core::RakeTask.new(:spec) do |t|
+  t.pattern = 'spec/**/*_spec.rb'
+end
+
+Cane::RakeTask.new(:quality) do |cane|
+  cane.canefile = '.cane'
+end

data/bin/sumo

@@ -1,72 +1,5 @@
-#!/usr/bin/ruby
+#!/usr/bin/env ruby
 
 require 'sumo'
-require 'sumo/version'
-require 'time'
-require 'optparse'
 
-include Sumo
-
-options = {
-  :query_options => {},
-  :config_file => DEFAULT_CONFIG_FILE,
-  :key => nil
-}
-
-OptionParser.new do |opts|
-  opts.banner = 'Usage: sumo [flags/options]'
-
-  opts.separator ''
-  opts.separator 'Specific options:'
-
-  opts.on('-q', '--query QUERY', 'Search for QUERY (required)') do |q|
-    options[:query] = q
-  end
-
-  opts.on('-f', '--from START', 'Begin at the given ISO8601 time') do |f|
-    options[:query_options]['from'] = f
-  end
-
-  opts.on('-t', '--to END', 'Finish at the given ISO8601 time') do |t|
-    options[:query_options]['to'] = t
-  end
-
-  opts.on('-z', '--time-zone ZONE', 'Use the given time zone') do |tz|
-    options[:query_options]['tz'] = tz
-  end
-
-  opts.on('-c', '--config-file FILE',
-          "Use the given config file instead of #{DEFAULT_CONFIG_FILE}") do |f|
-    options[:config_file] = config_file
-  end
-
-  opts.on('-e', '--extract-key KEY', 'Extract the specified key') do |k|
-    options[:key] = k
-  end
-
-  opts.on('-h', '--help', 'Print this message') do
-    puts opts
-    exit 0
-  end
-
-  opts.on('-v', '--version', 'Print the version') do
-    puts VERSION
-    exit 0
-  end
-end.parse!
-
-if options[:query].nil?
-  puts 'Please specify a query'
-  exit 1
-else
-  begin
-    puts search(options[:query],
-                options: options[:query_options],
-                key: options[:key],
-                config_file: options[:config_file])
-    exit 0
-  rescue StandardError => ex
-    puts ex.message
-    exit 1
-  end
-end
+Sumo::CLI.run

data/lib/sumo/cli.rb

@@ -0,0 +1,43 @@
+# This class is used to define a CLI.
+class Sumo::CLI < Clamp::Command
+  option ['-q', '--query'], 'QUERY', 'The query that will be sent to Sumo'
+  option ['-f', '--from'], 'FROM', 'The start time of the query (iso8601).'
+  option ['-t', '--to'], 'TO', 'The end time of the query (iso8601).'
+  option ['-z', '--time-zone'], 'TZ', 'The time zone of the FROM and TO times.'
+  option ['-e', '--extract-key'], 'KEY', 'The key to extract from the raw JSON.'
+  option ['-r', '--records'], :flag, 'Extract records instead of messages.'
+  option ['-v', '--version'], :flag, 'Print the version.'
+
+  # This method is called when the CLI is run.
+  def execute
+    if version?
+      puts Sumo::VERSION
+    elsif records?
+      search.records.each { |record| puts record }
+    else
+      search.messages.each { |message| puts format_message(message) }
+    end
+  rescue StandardError => ex
+    puts "#{ex.class}: #{ex.message}"
+    exit 1
+  end
+
+  def format_message(message)
+    if extract_key.nil?
+      message['_raw']
+    else
+      JSON.parse(message['_raw'])[extract_key]
+    end
+  end
+  private :format_message
+
+  def search
+    Sumo::Search.create(
+      :query => query,
+      :from => from,
+      :to => to,
+      :time_zone => time_zone
+    )
+  end
+  private :search
+end

data/lib/sumo/client.rb

@@ -0,0 +1,82 @@
+# This class has the lowest-level interface to interact with the Sumo Job API.
+class Sumo::Client
+  include Sumo::Error
+
+  attr_reader :creds, :cookie
+
+  # The error message raised when the result can be parsed from Sumo.
+  DEFAULT_ERROR_MESSAGE = 'Error sending API request'
+
+  # Create a new `Sumo::Client` with the given credentials.
+  def initialize(creds = Sumo.creds)
+    @creds = creds.freeze
+  end
+
+  # Send a HTTP request to the server, handling any errors that may occur.
+  def request(hash, &block)
+    response = connection.request(add_defaults(hash), &block)
+    handle_errors!(response)
+    set_cookie!(response)
+    response.body
+  end
+
+  # Define methods for the HTTP methods used by the API (#get, #post, and
+  # #delete).
+  [:get, :post, :delete].each do |http_method|
+    define_method(http_method) do |hash, &block|
+      request(hash.merge(:method => http_method), &block)
+    end
+  end
+
+  # Private functions that operate on the request and response.
+
+  def add_defaults(hash)
+    hash.merge(
+      :headers => default_headers.merge(hash[:headers] || {}),
+      :path => "/api/v#{Sumo::API_VERSION}#{hash[:path]}"
+    )
+  end
+  private :add_defaults
+
+  def handle_errors!(response)
+    case response.status
+    when 400..499 then raise ClientError, extract_error_message(response.body)
+    when 500..599 then raise ServerError, extract_error_message(response.body)
+    end
+  end
+  private :handle_errors!
+
+  def set_cookie!(response)
+    @cookie = response.headers['Set-Cookie'] || @cookie
+  end
+  private :set_cookie!
+
+  def extract_error_message(body)
+    JSON.parse(body)['message'] || DEFAULT_ERROR_MESSAGE
+  rescue
+    DEFAULT_ERROR_MESSAGE
+  end
+  private :extract_error_message
+
+  def default_headers
+    {
+      'Authorization' => "Basic #{encoded_creds}",
+      'Content-Type' => 'application/json',
+      'Cookie' => cookie,
+      'Accept' => 'application/json'
+    }.reject { |_, value| value.nil? }
+  end
+  private :default_headers
+
+  def encoded_creds
+    @encoded_creds ||= Base64.encode64(creds).strip
+  end
+  private :encoded_creds
+
+  def connection
+    @connection ||= Excon.new(
+      'https://api.sumologic.com'
+    )
+  end
+  private :connection
+end

data/lib/sumo/collection.rb

@@ -0,0 +1,97 @@
+# This class is used to un-paginate results from the API. Specifically, this
+# is currently used to page through records and messages returned by the API.
+class Sumo::Collection
+  include Enumerable
+  include Sumo::Error
+
+  attr_reader :offset
+
+  # Create a new collection.
+  def initialize(hash = {})
+    @offset = hash[:offset] || 0
+    @get_values = hash[:get_values]
+    @get_status = hash[:get_status]
+    @count_key = hash[:count_key]
+  end
+
+  # Iterate through each member of the collection, lazily making HTTP requests
+  # to get the next member. If no block is given, an `Enumerator` is returned.
+  def each(&block)
+    if block.nil?
+      enumerator
+    else
+      enumerator.each { |value| block.call(value) }
+      self
+    end
+  end
+
+  def enumerator
+    @enumerator ||= Enumerator.new do |values|
+      page.each { |value| values << value }
+      remaining.each { |value| values << value } if has_next_page?
+    end
+  end
+  private :enumerator
+
+  def values(hash)
+    @get_values.call(hash)
+  end
+  private :values
+
+  def status
+    @status ||= get_new_status
+  end
+  private :status
+
+  def get_new_status
+    stat = { 'state' => '', @count_key => @offset }
+    until (@offset < stat[@count_key]) || stat['state'].start_with?('DONE')
+      stat = @get_status.call
+      sleep 1
+    end
+    stat
+  end
+  private :get_new_status
+
+  def total
+    status[@count_key]
+  end
+  private :total
+
+  def state
+    status['state']
+  end
+  private :state
+
+  def page
+    @page ||= has_results? ? values(:offset => offset, :limit => limit) : []
+  end
+  private :page
+
+  def has_results?
+    limit > 0
+  end
+
+  def limit
+    @limit ||= begin
+      natural_limit = total - offset
+      (natural_limit <= 1000) ? natural_limit : 1000
+    end
+  end
+  private :limit
+
+  def has_next_page?
+    ['GATHERING RESULTS', 'NOT STARTED'].include?(state)
+  end
+  private :has_next_page?
+
+  def remaining
+    @remaining ||= Sumo::Collection.new(
+      :offset => offset + limit,
+      :get_values => @get_values,
+      :get_status => @get_status,
+      :count_key => @count_key
+    )
+  end
+  private :remaining
+end