logplex-client 0.1.4

data/lib/logplex/client/cli.rb

@@ -0,0 +1,219 @@
+require 'clamp'
+
+module Logplex
+  class Client
+
+    # The logplex client command-line interface
+    class CLI < Clamp::Command
+
+      # Generic error used for subclassing CLI-specific problems
+      class Error < StandardError; end
+
+      # General 'this command failed' error
+      class CommandFailed < Error; end
+
+      option ["-v", "--version"], :flag, "Show logplex client version" do
+        puts "logplex-client #{VERSION}"
+        exit 0
+      end
+
+      subcommand "channels:create", "Create a logplex channel" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "NAME", "channel name"
+        parameter "[TOKEN] ...", "associated channel tokens"
+        def execute
+          print "creating channel... "
+          channel = client.create_channel(name)
+          puts "done"
+
+          puts "channel id: #{channel.id}"
+          if !token_list.nil? && !token_list.empty?
+            token_list.each do |token| 
+              puts "creating token (#{token})"
+              channel.create_token(token)
+            end
+          end
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "channels:get", "Get channel info" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+        def execute
+          print "getting channel (#{channel})... "
+          chan = client.channel(channel)
+          puts "done"
+          puts
+          puts "id: #{chan.id}"
+          puts "no tokens" if chan.tokens.empty?
+          chan.tokens.each do |token|
+            puts "token: #{token.name} (#{token.id})"
+          end
+          puts "no drains" if chan.drains.empty?
+          chan.drains.each do |drain|
+            puts "drain: #{drain.drain_url} (#{drain.id})"
+          end
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "channels:delete", "Delete a channel and associated drains/tokens" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel name or id"
+        def execute
+          print "deleting channel... "
+          client.channel(channel).destroy
+          puts "done"
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "tokens:create", "Add a token to a channel" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+        parameter "NAME", "token name"
+        def execute
+          print "creating token... "
+          chan = client.channel(channel)
+          token = chan.create_token(name)
+          puts "done"
+          puts "#{name} => #{token.id}"
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "tokens:delete", "Remove a token from a channel" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+        parameter "NAME", "token name"
+        def execute
+          # TODO(sissel): Logplex doesn't appear to have (or document?) this functionality.
+          # per http://logplex.herokuapp.com/ as of 2012/04/20
+          $stderr.puts "Logplex currently does not implement this functionality"
+          return
+
+          #found = false
+          #client.channel(channel).tokens.each do |token|
+            #if token.name == name
+              #puts "deleting token... "
+              #token.destroy 
+              #found = true
+            #end
+          #end
+          #puts "no token found" if !found
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "drains:create", "Add a drain to a channel" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+
+        # For now, due to a bug (or misdocumented feature) in logplex, a drain 
+        # url is required at creation-time.
+        parameter "URL", "drain url (like 'syslog://..../')"
+
+        def execute
+          print "creating drain... "
+          chan = client.channel(channel)
+          drain = chan.create_drain(url)
+          puts "done"
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "drains:delete", "Remove a drain from a channel" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+        parameter "DRAIN", "drain id or token"
+        def execute
+          found = false
+          channel = client.channel(channel)
+          channel.drains.each do |d|
+            if d.token == drain or d.id == drain
+              print "deleting drain... "
+              d.destroy
+              puts "done"
+              found = true
+            end
+          end
+          puts "no drain found" if !found
+        rescue => e
+          failure(e)
+        end
+      end
+
+      subcommand "channels:logs", "fetch channel logs" do
+        option "--logplex-url", "LOGPLEX_URL",
+          "The url to logplex, like 'https://user:pass@logplex.example.com/'",
+          :environment_variable => "LOGPLEX_URL", :required => true
+        parameter "CHANNEL", "channel id"
+        option ["--source", "-s"], "SOURCE", "log source"
+        option ["--ps", "-p"], "PS", "log process"
+        option ["--num", "-n"], "NUM", "max number of logs to fetch"
+        option ["--tail", "-t"], :flag, "maintain a tail session of log stream"
+        option ["--chunk-size", "-c"], "CHUNKS_SIZE", "response chunk size", :default => Logplex::Session::DEFAULT_CHUNK_SIZE
+
+        def execute
+          opts = {}
+          opts[:source] = source if source
+          opts[:ps] = ps if ps
+          opts[:num] = num.to_i if num
+          opts[:tail] = true if tail?
+          opts[:chunk_size] = chunk_size.to_i if chunk_size
+          chan = client.channel(channel)
+          session = chan.create_session(opts)
+          session.each_event do |event|
+            puts event
+          end
+        rescue => e
+          failure(e)
+        end
+      end
+
+      private
+
+      # Private: Get a logplex client instance.
+      # 
+      # Subsequent calls to this method wil return the same object.
+      #
+      # Returns a Logplex::Client
+      def client
+        @client ||= Logplex::Client.new(logplex_url)
+      end # def client
+
+      # Private: Emit a failure.
+      def failure(e)
+        puts "failed"
+
+        if e.is_a?(Logplex::Client::Backends::HTTP::NotFound)
+          puts "!    error: Object not found (#{e.to_s})"
+        else
+          puts "!    error: #{e.to_s}"
+        end
+        exit(1)
+      end # def failure
+    end # class Logplex::Client::CLI
+  end # class Logplex::Client
+end # module Logplex

data/lib/logplex/client/version.rb

@@ -0,0 +1,6 @@
+module Logplex
+  class Client
+    # The version of this library.
+    VERSION = "0.1.4"
+  end
+end

data/lib/logplex/client.rb

@@ -0,0 +1,80 @@
+require "logplex/namespace"
+require "logplex/client/version"
+require "logplex/client/backends/http"
+require "logplex/channel"
+require "logplex/session"
+
+# Public: a logplex client.
+#
+# This class allows you to access logplex as a client; reading and writing
+# logs, creating channels, tokens, etc.
+#
+# For logplex's HTTP API, see http://logplex.herokuapp.com
+#
+# Examples
+#
+#   client = Logplex::Client.new("https://user:pass@logplex.example.com/")
+#   channel = client.create_channel("my-example-channel")
+#   TODO(sissel): Complete this example.
+class Logplex::Client
+  # Public: Initialize a client pointing at a given url.
+  #
+  # url - a url (including user+pass) to your logplex service
+  #
+  # Examples
+  #
+  #   Logplex::Client.new("https://user:pass@logplex.heroku.com/")
+  def initialize(url)
+    @url = url
+    @backend = Logplex::Client::Backends::HTTP.new(@url)
+  end # def initialize
+
+  # Public: Create a channel.
+  #
+  # Note: the 'name' of the channel lives in a global namespace. Choose wisely.
+  # Note: You must save the channel id if you wish to use the channel later.
+  #
+  # name - the string name to give to the channel being created.
+  #
+  # Examples
+  # 
+  #   client.create_channel("my-example-channel")
+  #
+  # Returns a {Logplex::Channel}
+  # Raises TODO(sissel): Raises what?
+  def create_channel(name)
+    # TODO(sissel): Call the API
+    result = @backend.create_channel(name)
+    return Logplex::Channel.new(@url, result[:channel_id])
+  end # def create_channel
+
+  # Public: Get a {Logplex::Channel} instance with a given id.
+  #
+  # If you have already created a channel, this is how you access it later.
+  #
+  # channel_id - the channel id (number)
+  #
+  # Returns a {Logplex::Channel}
+  # Raises TODO(sissel): ??? if the channel is not found
+  def channel(channel_id)
+    # This will throw an exception if it doesn't exist.
+    result = @backend.get_channel(channel_id)
+    chan = Logplex::Channel.new(@url, channel_id)
+    # Hack for now to push the result of get_channel into the Channel.
+    chan.instance_eval { @info = result } 
+    return chan
+  end # def channel
+
+  # Public: Get a {Logplex::Session} instance with a given id.
+  #
+  # If you have already created a session, this is how you access it later.
+  #
+  # session_id - the session id (number)
+  #
+  # Returns a {Logplex::Session}
+  # Raises TODO(sissel): ??? if the session is not found
+  def session(session_id)
+    raise NotImplemented
+  end # def session
+end # class Logplex::Client
+

data/lib/logplex/drain.rb

@@ -0,0 +1,45 @@
+require "logplex/namespace"
+
+# Operations on a Logplex::Drain:
+#   Add a url to a drain: POST /v2/channels/<channel>/drains/<drain> { "url": ... }
+#     drain.use(url) ### I like 'use' better than 'drain.url = "url"'
+#   Delete a drain: DELETE /v2/channels/<channel>/drains/<drain>
+#     drain.destroy
+#
+class Logplex::Drain
+  def initialize(url, channel_id, drain_id, drain_token, drain_url=nil)
+    @url = url
+    @channel_id = channel_id
+    @id = drain_id
+    @token = drain_token
+    @drain_url = drain_url
+    @backend = Logplex::Client::Backends::HTTP.new(@url)
+  end # def initialize
+
+  # Public: Destroy this drain.
+  #
+  # Returns nothing
+  # Raises TODO(sissel): document exceptions
+  def destroy
+    @backend.delete_drain(@channel_id, @drain_id)
+  end # def destroy
+
+  # Public: Set the output of this drain to the given drain_url.
+  #
+  # drain_url - the url to drain logs to. Usually of form syslog://host:port/
+  #
+  # Returns nothing
+  # Raises TODO(sissel): Document exceptions
+  def drain_url=(drain_url)
+    # I don't really like this API feel, doing RPC calls in a setter feels like
+    # it breaks expectations.
+    @backend.set_drain_url(@channel_id, @drain_id, drain_url)
+    @target = drain_url
+  end # def use
+
+  # Public: Get the url for this drain.
+  attr_reader :drain_url
+
+  # Public: Get the id for this drain
+  attr_reader :id
+end # class Logplex::Drain

data/lib/logplex/emitter.rb

@@ -0,0 +1,79 @@
+require "logplex/namespace"
+
+# A logging emitter. Best to create this with {Logplex::Token#emitter}
+class Logplex::Emitter
+
+  # Default methods to private, see bottom of class for public method
+  # declarations
+  private
+
+  # Public: Create an emitter with the given target address and token.
+  def initialize(address, token)
+    @address = address
+    @token = token
+
+    connect
+  end # def initialize
+
+  # Public: Emit an event.
+  # 
+  # procid - the process id emitting this message. A string, like "web.1"
+  # message - the message to emit in this event.
+  def emit(procid, message)
+    #   FRAME: LENGTH PAYLOAD
+    #   LENGTH: decimal value of the length of the payload
+    payload = serialize_syslogish(13, Time.now.strftime("%Y-%m-%dT%H:%M:%S%z"),
+                                  Socket.gethostname, @token, procid, 
+                                  # 'message id' is meaningless to us
+                                  "-", 
+                                  # This extra "- " is due to a bug in logplex.
+                                  "- #{message}")
+    event = "#{payload.size} #{payload}"
+    @socket.syswrite(event)
+  end # def emit
+
+
+  # Private: connect to the remote address
+  #
+  # Returns nothing
+  # Raises TODO(sissel): ???
+  def connect
+    # TODO(sissel): Handle errors/reconnections
+    @socket.close unless @socket.nil?
+    host, port = @address.split(":")
+    @socket = TCPSocket.new(host, port.to_i)
+  end # def connect
+
+  # Private: serialize an RFC5424 message given some parameters
+  #
+  # pri - the syslog priority (this is facility combined with severity, number)
+  # timestamp - the RFC3339 timestamp (string)
+  # host - the hostname generating this message (string)
+  # appname - the application generating this message (string)
+  # procid - the process's id who is generating this message (string)
+  # msgid - the message id, usually just "-" (string)
+  # msg - the message (string)
+  #
+  # Returns the string encoding of an RFC5424 message based on the given
+  # arguments
+  def serialize_syslogish(pri, timestamp, host, appname, procid, msgid, msg)
+    # This protocol is RFC5424 layered on the message framing portion of
+    # RFC5425 (no ssl/tls)
+    #
+    # This method implements only the event formatting, not the framing.
+    #
+    # It is roughly:
+    #   PAYLOAD: <PRI>VERSION TIMESTAMP HOST APPNAME PROCID MSGID MSG
+    #   PRI: 0-191 (syslog pri combination of severity * 8 + facility
+    #   VERSION: 1
+    #   TIMESTAMP: RFC3339 time
+    #   HOST: hostname
+    #   APPNAME: the logplex channel token (t.<UUID>)
+    #   PROCID: the process id, on heroku this is usually something like 'web.9'
+    #   MSGID: usually null as "-"
+    return ["<#{pri}>1", timestamp, host, appname, procid, msgid, msg].join(" ")
+  end # def serialize_syslogish
+
+  # Explicitly declare methods public.
+  public(:initialize, :emit)
+end # class Logstash::Emitter

data/lib/logplex/event.rb

@@ -0,0 +1,8 @@
+require "logplex/namespace"
+
+# Public: An event received from a logplex session.
+#
+# This class is not well-thought-out yet.
+class Logplex::Event
+  attr_accessor :time, :token, :process, :message
+end # class Logplex::Event

data/lib/logplex/namespace.rb

@@ -0,0 +1,8 @@
+# :nodoc:
+module Logplex
+  # :nodoc:
+  class Client
+    # :nodoc:
+    module Backends; end
+  end
+end # module Logplex

data/lib/logplex/session.rb

@@ -0,0 +1,72 @@
+require "logplex/namespace"
+require "excon"
+
+# Operations on a Logplex::Session:
+#   Get logs for a session - GET /sessions/<session>
+#     API> session.each_event do |event|
+#            # 'event' should be a nice object representation, not a string.
+#            of a session
+#          end
+#
+
+# Public: A logplex event session.
+#
+# This class allows you to read events from logplex.
+#
+# You generally acquire an instance of this class through
+# Logplex::Channel#create_session
+class Logplex::Session
+  DEFAULT_CHUNK_SIZE = 1024
+
+  # Public: Initialize a logplex session.
+  #
+  # url - the url to the session, usually of the form: 
+  #   https://user:pass@logplex/sessions/session-id
+  #
+  # session_settings - a hash of arguments; see
+  #   {Logplex::Channel#create_session} for values.
+  def initialize(url, session_settings)
+    @url = url
+
+    if session_settings[:tail]
+      @tail = true
+    else
+      @tail = false
+      @limit = session_settings[:num]
+    end
+    @chunk_size = session_settings[:chunk_size] || DEFAULT_CHUNK_SIZE
+  end # def initialize
+
+  # Public: iterate over events received from this logplex session
+  #
+  # Yields Logplex::Event objects, one per event.
+  # Returns nothing
+  # Raises TODO(sissel): ???
+  def each_event(&block)
+    connection = Excon.new(@url, :chunk_size => @chunk_size)
+
+    @buffer = ""
+    @event_count = 0
+
+    process_response = lambda do |chunk, remaining_bytes, total_bytes|
+      @buffer += chunk
+
+      events = @buffer.split("\n")
+      if @buffer[-1] != "\n"
+        @buffer = events.last
+        events = events[0 .. -2]
+      else
+        @buffer = ""
+      end
+
+      events.each do |line|
+        @event_count += 1
+        block.call(line)
+        return if !@tail && @limit < @event_count
+      end
+    end
+
+    response = connection.get(:response_block => process_response)
+    raise "Error: #{response.body}" if response.status != 200
+  end # def each_block
+end # class Logplex::Session

data/lib/logplex/token.rb

@@ -0,0 +1,42 @@
+require "logplex/namespace"
+require "logplex/emitter"
+
+# Operations on a Logplex::Token:
+#   Publish an event:
+#     API> token.publish(message)
+#
+#
+
+# A logplex token is required for publishing events to logplex.
+#
+# To create a token, see Logplex::Channel#create_token.
+class Logplex::Token
+  # Public: initialize a Logplex::Token
+  #
+  # url - the url to logplex; see Logplex::Client.new for more info
+  # channel_id - the channel id to create a token on; see Logplex::Channel for
+  #   more info
+  # name - the string name of this token. This name will appear in logplex
+  #   output (drains and log sessions) as the application name.
+  # token_id - the token; usually in the form 't.SOME-UUID-VALUE'
+  def initialize(url, channel_id, name, token_id)
+    @url = url
+    @channel_id = channel_id
+    @name = name
+    @id = token_id
+  end # def initialize
+
+  attr_accessor :name
+  attr_accessor :id
+  attr_reader :channel_id
+  attr_reader :url
+
+  # Public: Get an emitter suitable for publishing events with this token.
+  #
+  # Returns a Logplex::Emitter
+  def emitter
+    address = [URI.parse(@url).host, 601].join(":")
+    @emitter ||= Logplex::Emitter.new(address, @id)
+    return @emitter
+  end # def emitter
+end # class Logplex::Token

data/logplex-client.gemspec

@@ -0,0 +1,30 @@
+# -*- encoding: utf-8 -*-
+
+require File.expand_path("../lib/logplex/client/version", __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = ["Jacob Vorreuter", "Jordan Sissel"]
+  gem.email         = ["jacob.vorreuter@gmail.com", "jls@heroku.com"]
+  gem.description   = %q{A client and library for Logplex}
+  gem.summary       = %q{A client and library for Logplex}
+  gem.homepage      = "https://github.com/heroku/logplex-client"
+
+  gem.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  gem.files         = `git ls-files`.split("\n")
+  gem.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  gem.name          = "logplex-client"
+  gem.require_paths = ["lib"]
+  gem.version       = Logplex::Client::VERSION
+
+  # For command line flag parsing, etc.
+  gem.add_dependency "clamp", "~> 0.4.0"
+
+  # For json parsing
+  gem.add_dependency "multi_json"
+
+  # For HTTP stuff that RestClient can't handle
+  gem.add_dependency "excon", "~> 0.20.1"
+
+  # For general HTTP access to REST-ish APIs
+  gem.add_dependency "rest-client"
+end

data/spec/documentation_spec.rb

@@ -0,0 +1,41 @@
+require "rubygems"                                                                                               
+require "yard"                                                                                                   
+
+class MissingDocumentation < StandardError; end
+
+describe "this project" do
+  before do
+    # Use YARD to parse all ruby files found in '../lib'
+    libdir = File.join(File.dirname(__FILE__), "..", "lib")
+    YARD::Registry.load(Dir.glob(File.join(libdir, "**", "*.rb")))
+    @registry = YARD::Registry.all
+  end
+
+  it "must have all classes, modules, and constants documented" do
+    # YARD's parser works best in ruby 1.9.x, so skip 1.8.x
+    skip if RUBY_VERSION < "1.9.2"
+    # Note, the 'find the undocumented things' code here is 
+    # copied mostly from: YARD 0.7.5's lib/yard/cli/stats.rb
+    #
+    # Find all undocumented classes, modules, and constants
+    undocumented = @registry.select do |o| 
+      [:class, :module, :constant].include?(o.type) && o.docstring.blank?
+    end
+
+    # Find all undocumented methods
+    methods = @registry.select { |m| m.type == :method }
+    methods.reject! { |m| m.is_alias? || !m.is_explicit? }
+    undocumented += methods.select do |m| 
+      m.docstring.blank? && !m.overridden_method
+    end
+
+    if (undocumented.length > 0)
+      message = ["The following are not documented"]
+      undocumented.each do |o|
+        message << "* #{o.type.to_s} #{o.to_s} <#{o.file}:#{o.line}>"
+      end
+
+      raise MissingDocumentation.new(message.join("\n"))
+    end
+  end
+end