logplex-client 0.1.4

data/.gitignore

@@ -0,0 +1,4 @@
+*.swp
+.yardoc/
+doc/
+Gemfile.lock

data/Gemfile

@@ -0,0 +1,13 @@
+source :rubygems
+
+gemspec
+
+group :documentation do
+  gem "yard"
+  gem "yard-tomdoc"
+end
+
+group :testing do
+  gem "rspec"
+  gem "insist"
+end

data/Makefile

@@ -0,0 +1,34 @@
+GEMSPEC=$(shell ls *.gemspec | head -1)
+VERSION=$(shell ruby -rubygems -e 'puts Gem::Specification.load("$(GEMSPEC)").version')
+PROJECT=$(shell ruby -rubygems -e 'puts Gem::Specification.load("$(GEMSPEC)").name')
+GEM=$(PROJECT)-$(VERSION).gem
+
+.PHONY: test
+test:
+	bundle exec rspec
+
+.PHONY: package
+package: $(GEM)
+
+# Always build the gem
+.PHONY: $(GEM)
+$(GEM):
+	gem build $(PROJECT).gemspec
+
+showdocs:
+	yard server --plugin yard-tomdoc -r
+
+clean:
+	-rm -r .yardoc/ doc/ *.gem 
+
+.PHONY: install
+install: $(GEM)
+	gem install $<
+
+# Publish to gemgate
+.PHONY: publish
+publish: APPNAME=gemgate-heroku-internal-gems
+publish: URL=https://$(APPNAME).herokuapp.com
+publish: GEMGATE_AUTH=$(shell heroku config -a $(APPNAME) | awk '/GEMGATE_AUTH/ { print $$NF }')
+publish: $(GEM)
+	curl -F file=@$(GEM) -u $(GEMGATE_AUTH) $(URL)

data/README.md

@@ -0,0 +1,117 @@
+## Logplex Client
+
+### command line interface
+
+What follows are examples.
+
+```bash
+# You must do this.
+$ export LOGPLEX_URL=https://heroku:password@logplex.heroku.com
+
+$ bin/logplex channels:create channel-blah
+creating channel... done
+channel id: 3858972
+
+$ logplex channels:get 3858972
+getting channel (3858972)... done
+
+id: 3858972
+no tokens
+no drains
+
+$ logplex channels:delete 3858972
+deleting channel... done
+
+$ logplex tokens:create 3858972 helloworld
+creating token...
+helloworld => t.7c55b81f-59b3-45bb-87b1-aefc0fd96a40
+
+$ bin/logplex channels:get 3858972
+getting channel (3858972)...done
+
+id: 3858972
+token: helloworld (t.7c55b81f-59b3-45bb-87b1-aefc0fd96a40)
+no drains
+
+$ bin/logplex channels:logs 3858972 --ps web.1
+2012-03-22T00:12:47+00:00 app[web.1]: foo
+2012-03-22T00:12:48+00:00 app[web.1]: bar
+```
+
+## Ruby Usage
+
+### ruby api/usage docs
+
+You can view the rubydoc for this project by running:
+
+```bash
+$ bundle install
+$ make showdocs
+```
+
+Then point your browser at <http://localhost:8808/> to view the HTML version of
+the API docs.
+
+### Managing Logplex
+
+```ruby
+# Get a new client.
+=> client = Logplex::Client.new("https://heroku:password@logplex.heroku.com")
+
+# Create a named channel.
+=> channel = client.create_channel("channel-blah")
+
+# Get a known channel by id
+=> channel = client.channel(1001)
+
+# Destroy a channel
+=> channel.destroy
+
+# Create a named token on a channel
+=> token = channel.create_token("app")
+
+# Destroy a token
+=> token.destroy
+
+# Create a drain
+=> drain = channel.create_drain
+
+# Get a known drain by a drain id
+=> drain = channel.drain(12345)
+
+# Make the drain point at a specific syslog receiver:
+=> drain.url = "syslog://example.com:1234/
+
+# Destroy a drain
+=> drain.destroy
+```
+
+### Writing to Logplex
+
+```ruby
+# Get an Emitter for writing logs using a token
+=> emitter = token.emitter
+
+# Writing an event
+# Note, you must be running on the heroku platform for this to work as
+# currently the port used by log transport into logplex is firewalled.
+=> emitter.emit("processname", "message")
+```
+
+### Reading from Logplex
+
+```ruby
+# Create a log session (for reading logs)
+=> session = channel.session(:num => 10)
+# or tail things
+=> session = channel.session(:num => 10, :tail => true
+
+# Read logs from a session
+=> session.each_event do |event|
+     puts event
+   end
+```
+
+### Logplex HTTP API
+
+https://logplex.herokuapp.com/

data/bin/logplex

@@ -0,0 +1,12 @@
+#!/usr/bin/env ruby
+
+require "logplex/client"
+require "logplex/client/cli"
+
+begin
+  Logplex::Client::CLI.run
+rescue Errno::EPIPE
+#rescue Logplex::Client::Error => e
+  #$stderr.puts " !    #{e}"
+  #exit 1
+end

data/lib/logplex/channel.rb

@@ -0,0 +1,144 @@
+require "logplex/namespace"
+require "logplex/drain"
+require "logplex/session"
+require "logplex/token"
+
+# Operations on a Logplex::Channel:
+#   Delete a channel - DELETE /v2/channels/<channel>
+#     API> channel.destroy => nil
+#   Get channel info - GET /v2/channels/<channel>
+#     API> channel.tokens => [Token, Token, ...]
+#     API> channel.drains => [Drain, Drain, ...]
+#   Get a token with known name: (just a wrapper of 'channel.tokens')
+#     API> channel.token(name) => Token
+#   Create a token - POST /v2/channels/<channel>/tokens { "name": "the token name" }
+#     API> channel.create_token(name) => Token
+#   Create a drain - POST /v2/channels/<channel>/drains { "url": "syslog://.../" }
+#     API> channel.create_drain(url) => Drain
+#   Create a session - POST /v2/sessions { "channel_id": <channel>, ... }
+#     API> channel.create_session(:source => ..., ps => ..., num => ..., tail => ...)  => Session
+
+# Public: A logplex channel.
+#
+# You generally acquire one of these objects from
+# Logplex::Client#create_channel or Logplex::Client#channel
+class Logplex::Channel
+  # Public: initialize a channel object
+  #
+  # url - the url to logplex. See Logplex::Client#new for more info.
+  # channel_id - the channel id, should be a number.
+  #
+  # See also: Logplex::Client#create_channel and Logplex::Client#channel
+  def initialize(url, channel_id)
+    @url = url
+    @channel_id = channel_id
+    @backend = Logplex::Client::Backends::HTTP.new(@url)
+  end # def initialize
+
+  # Public: Destroy a channel.
+  #
+  # This will remove a channel from logplex as well as any associated tokens or
+  # drains.
+  #
+  # Returns nothing.
+  # Raises # TODO(sissel): What errors?
+  def destroy
+    @backend.delete_channel(@channel_id)
+  end # def destroy
+
+  # Public: Get the channel id.
+  #
+  # Returns the channel id.
+  def id
+    return @channel_id
+  end # def id
+
+  # Public: Get a list of tokens associated with this channel.
+  #
+  # Note: To create a token, see Logplex::Channel#create_token
+  #
+  # Returns an Array of Logplex::Tokens
+  # Raises # TODO(sissel): ???
+  def tokens
+    @info ||= @backend.get_channel(@channel_id)
+    result = @info
+    return result[:tokens].collect do |info| 
+      Logplex::Token.new(@url, @channel_id, info[:name], info[:token])
+    end
+  end # def tokens
+
+  # Public: Get a list of drains associated with this channel.
+  #
+  # Note: To create a drain, see Logplex::Channel#create_drain
+  #
+  # Returns an Array of Logplex::Drains
+  # Raises # TODO(sissel): ???
+  def drains
+    @info ||= @backend.get_channel(@channel_id)
+    result = @info
+    return result[:drains].collect do |info|
+      Logplex::Drain.new(@url, @channel_id, info[:id], info[:token], info[:url])
+    end
+  end # def drains
+
+  # Public: Create a new token for this channel.
+  #
+  # name - a String name for this token. The name is meaningful and will 
+  #   appear in drain or session output.
+  #
+  # Returns a Logplex::Token
+  def create_token(name)
+    result = @backend.create_token(@channel_id, name)
+    return Logplex::Token.new(@url, @channel_id, result[:name], result[:token])
+  end # def create_token
+
+  # Public: Create a new drain for this channel.
+  #
+  # For information on what a drain is, see Logplex::Drain
+  #
+  # Returns a Logplex::Drain
+  # Raises # TODO(sissel): ???
+  def create_drain(url=nil)
+    # Just create the drain, don't add any URLs to it.
+    # API call /channels/[channel-id]/drains/tokens
+    result = @backend.create_drain(@channel_id, url)
+    return Logplex::Drain.new(@url, @channel_id, result[:id], result[:token], url)
+  end # def create_drain
+
+  # Public: Create a new session for this channel.
+  #
+  # For information on what a session is, see Logplex::Session
+  #
+  # settings - a hash containing:
+  #   :source - the token name, like "app" or "heroku" (default: nil, optional)
+  #   :ps - the RFC5424 process id, like "web.1" (default: nil, optional)
+  #   :num - the number of events to retrieve (default: 40, optional)
+  #   :tail - true if you wish to follow the live event stream 
+  #     (default: false, optional)
+  #
+  # Returns a Logplex::Session
+  # Raises # TODO(sissel): ???
+  def create_session(settings={})
+    settings = { 
+      :num => 40,
+      :tail => false
+    }.merge(settings)
+
+    # Currently a bug(?) in logplex that if you specify 'num' = 0, you get all
+    # logs. TODO(sissel): If you specify '0.1' you get zero results, so use
+    # that hack to hide the '0' bug until the bug is fixed.
+    settings[:num] = 0.1 if settings[:num] == 0
+  
+    # Here's what core does: https://github.com/heroku/core/blob/master/lib/logplex.rb#L12
+    # TODO(sissel): generate a random 'srv' id
+    server_id = rand(1000)
+    result = @backend.create_session(@channel_id, server_id, settings)
+    # the foo=bar& prefix is required because of some problem with haproxy or
+    # how we are using it to pin http requests to backends.
+    # Additionally, sesssions also don't need authentication, so strip that out.
+    url = URI.parse(@url + result[:url] + "?" + "foo=bar&srv=#{server_id}")
+    url.user = nil
+    url.password = nil
+    return Logplex::Session.new(url.to_s, settings)
+  end # def create_session
+end # class Logplex::Channel

data/lib/logplex/client/backends/http.rb

@@ -0,0 +1,192 @@
+require "multi_json"
+require "restclient"
+require "logplex/namespace"
+require "openssl"
+
+# A client implementation that speaks to the logplex http interface.
+class Logplex::Client::Backends::HTTP
+  # Parent class of all Logplex-related HTTP errors.
+  class Error < StandardError; end
+
+  # An error indicating the call to Logplex failed due to authentication
+  # problems.
+  class Unauthorized < Error; end
+
+  # An error indicating the call to Logplex failed due to a HTTP request
+  # returning 404.
+  class NotFound < Error; end
+
+  # A new logplex backend that uses HTTP
+  #
+  # url - (string) url to logplex, like "https://user:pass@logplex.example.com/"
+  def initialize(url)
+    @url = url
+  end # def initialize
+
+  # Returns a hash with form: 
+  #   {:channel_id=>3639270, :tokens=>[]}
+  def create_channel(name, tokens = [])
+    json_post("/channels", {:name => name, :tokens => tokens})
+  end # def create_channel
+
+  # Public: Creates a channel.
+  #
+  # Returns a hash of the json response in the form:
+  #   {:channel_id=>3639270, :tokens=>[], :drains=>[]}
+  def get_channel(channel_id) 
+    json_get("/v2/channels/#{channel_id}")
+  end # def get_channel
+
+  # Public: Deletes a channel
+  def delete_channel(channel_id)
+    delete("/v2/channels/#{channel_id}")
+  end # def delete_channel
+
+  # Public: Creates a token on a given channel
+  def create_token(channel_id, token_name)
+    json_post("/v2/channels/#{channel_id}/tokens", {:name => token_name})
+  end # def create_token
+
+  # Public: Creates a drain for a given channel.
+  def create_drain(channel_id, url)
+    json_post("/v2/channels/#{channel_id}/drains", :url => url)
+  end # def create_drain
+
+  # Public: Sets the drain url for a channel+drain.
+  #
+  # channel_id - the channel id (number)
+  # drain_id - the drain id (string, given from {create_drain})
+  # url - the url to drain events to, like "syslog://host:port/"
+  def set_drain_url(channel_id, drain_id, url)
+    json_post("/v2/channels/#{channel_id}/drains/#{drain_id}", :url => url)
+  end # def set_drain_url
+
+  # Public: Delete a drain
+  # 
+  # channel_id - the channel id (number)
+  # drain_id - the drain id (string, given from {create_drain})
+  def delete_drain(channel_id, drain_id)
+    delete("/v2/channels/#{channel_id}/drains/#{drain_id}")
+  end
+
+  # Creates a event stream session on a channel.
+  #
+  # channel_id - the channel id (number)
+  # server_id - the server id to use in this request
+  #
+  # Note: The 'server_id' is just used for request routing purposes. When
+  # creating and reading a session, you *must* use the same server_id in both
+  # requests. This lets any load balancer in front of logplex pin the session
+  # create and read to the same backend logplex session.
+  def create_session(channel_id, server_id, opts = {})
+    # The 'foo=bar&' part is due to how we use haproxy for pinning
+    # Also channel_id should be a string, also.
+    opts = opts.merge(:channel_id => channel_id.to_s)
+    # num needs to be a string
+    opts[:num] = opts[:num].to_s if opts.has_key?(:num)
+    # If the 'tail' opt is false, remove it.
+    opts.delete(:tail) if !opts[:tail]
+    json_post("/v2/sessions?foo=bar&srv=#{server_id}", opts)
+  end # def create_session
+
+  private
+
+  # Private: Do an http GET an expect JSON response.
+  #
+  # Returns the ruby-equivalent object of the JSON data.
+  def json_get(path, options = {})
+    options = { :symbolize_keys => true }.merge(options)
+    options.update(:accept => "application/json")
+    symbolize_keys = options.delete(:symbolize_keys)
+
+    body = get(path, options)
+    MultiJson.load(body, :symbolize_keys => symbolize_keys)
+  end # def json_get
+
+  # Private: Do an http POST an expect JSON response.
+  #
+  # Returns the ruby-equivalent object of the JSON data.
+  def json_post(path, body = {}, options = {})
+    options = { :symbolize_keys => true }.merge(options)
+    options.update(:accept => "application/json")
+    symbolize_keys = options.delete(:symbolize_keys)
+
+    resp = post(path, MultiJson.dump(body), options)
+    MultiJson.load(resp, :symbolize_keys => symbolize_keys)
+  end # def json_post
+
+  # Private: Do an http GET.
+  #
+  # Passes path and options to RestClient without modification.
+  #
+  # Returns the response body.
+  # Raises NotFound on 404
+  # Raises Unauthorized on authentication failure
+  # Raises Error on any other failure (socket error, etc)
+  def get(path, options = {})
+    with_error_translation(path) do
+      connection[path].get(options)
+    end
+  end
+
+  # Private: Do an http POST.
+  #
+  # Passes path and options to RestClient without modification.
+  #
+  # Returns the response body.
+  # Raises NotFound on 404
+  # Raises Unauthorized on authentication failure
+  # Raises Error on any other failure (socket error, etc)
+  def post(path, body, options = {})
+    with_error_translation(path) do
+      connection[path].post(body, options)
+    end
+  end # def post
+
+  # Private: Do an http DELETE.
+  #
+  # Passes path and options to RestClient without modification.
+  #
+  # Returns the response body.
+  # Raises NotFound on 404
+  # Raises Unauthorized on authentication failure
+  # Raises Error on any other failure (socket error, etc)
+  def delete(path, options = {})
+    with_error_translation(path) do
+      connection[path].delete(options)
+    end
+  end # def delete
+
+  # Private: Call the block and wrap expected exceptions with the http path.
+  #
+  # Returns the result of the block on success.
+  def with_error_translation(path)
+    yield
+  rescue RestClient::ResourceNotFound => e
+    raise NotFound, "The path `#{path}` could not be found."
+  rescue RestClient::Unauthorized
+    raise Unauthorized, "Logplex authentication failed"
+  rescue RestClient::Exception, SystemCallError, SocketError => e
+    message = e.respond_to?(:http_code) ? e.http_code : "#{e.class}: #{e.message}"
+    message = "(#{message})" if message
+    raise Error, "Trouble communicating with Logplex. Try again soon. #{message}".strip
+  end
+
+  # Private Get an instance of RestClient::Resource to use as the http interface.
+  #
+  # Subsequent calls to this will return the same object instance.
+  #
+  # Returns an instnace of RestClient::Resource
+  def connection
+    @connection ||= RestClient::Resource.new(@url,
+                                             :verify_ssl => ssl_verification() )
+  end # def connection
+
+  def ssl_verification
+    if ENV['SSL_VERIFY_NONE'] then
+      OpenSSL::SSL::VERIFY_NONE
+    else
+      OpenSSL::SSL::VERIFY_PEER
+    end
+  end
+end # class Logplex::Client::Backends::HTTP