logstash-lib 1.3.2

data/lib/logstash/plugin.rb

@@ -0,0 +1,150 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "cabin"
+
+class LogStash::Plugin
+  attr_accessor :params
+  attr_accessor :logger
+
+  public
+  def hash
+    params.hash ^
+    self.class.name.hash
+  end
+
+  public
+  def eql?(other)
+    self.class.name == other.class.name && @params == other.params
+  end
+
+  public
+  def initialize(params=nil)
+    @params = params
+    @logger = Cabin::Channel.get(LogStash)
+  end
+
+  # This method is called when someone or something wants this plugin to shut
+  # down. When you successfully shutdown, you must call 'finished'
+  # You must also call 'super' in any subclasses.
+  public
+  def shutdown(queue)
+    # By default, shutdown is assumed a no-op for all plugins.
+    # If you need to take special efforts to shutdown (like waiting for
+    # an operation to complete, etc)
+    teardown
+    @logger.info("Received shutdown signal", :plugin => self)
+
+    @shutdown_queue = queue
+    if @plugin_state == :finished
+      finished
+    else
+      @plugin_state = :terminating
+    end
+  end # def shutdown
+
+  # You should call this method when you (the plugin) are done with work
+  # forever.
+  public
+  def finished
+    # TODO(sissel): I'm not sure what I had planned for this shutdown_queue
+    # thing
+    if @shutdown_queue
+      @logger.info("Sending shutdown event to agent queue", :plugin => self)
+      @shutdown_queue << self
+    end
+
+    if @plugin_state != :finished
+      @logger.info("Plugin is finished", :plugin => self)
+      @plugin_state = :finished
+    end
+  end # def finished
+
+  # Subclasses should implement this teardown method if you need to perform any
+  # special tasks during shutdown (like flushing, etc.)
+  public
+  def teardown
+    # nothing by default
+    finished
+  end
+
+  # This method is called when a SIGHUP triggers a reload operation
+  public
+  def reload
+    # Do nothing by default
+  end
+
+  public
+  def finished?
+    return @plugin_state == :finished
+  end # def finished?
+
+  public
+  def running?
+    return @plugin_state != :finished
+  end # def finished?
+
+  public
+  def terminating?
+    return @plugin_state == :terminating
+  end # def terminating?
+
+  public
+  def to_s
+    return "#{self.class.name}: #{@params}"
+  end
+
+  protected
+  def update_watchdog(state)
+    Thread.current[:watchdog] = Time.now
+    Thread.current[:watchdog_state] = state
+  end
+
+  protected
+  def clear_watchdog
+    Thread.current[:watchdog] = nil
+    Thread.current[:watchdog_state] = nil
+  end
+
+  public
+  def inspect
+    if !@config.nil?
+      description = @config \
+        .select { |k,v| !v.nil? && (v.respond_to?(:empty?) && !v.empty?) } \
+        .collect { |k,v| "#{k}=>#{v.inspect}" }
+      return "<#{self.class.name} #{description.join(", ")}>"
+    else
+      return "<#{self.class.name} --->"
+    end
+  end
+
+  # Look up a plugin by type and name.
+  public
+  def self.lookup(type, name)
+    # Try to load the plugin requested.
+    # For example, load("filter", "grok") will try to require
+    #   logstash/filters/grok
+    #
+    # And expects to find LogStash::Filters::Grok (or something similar based
+    # on pattern matching
+    path = "logstash/#{type}s/#{name}"
+    require(path)
+
+    base = LogStash.const_get("#{type.capitalize}s")
+    klass = nil
+    #klass_sym = base.constants.find { |c| c.to_s =~ /^#{Regexp.quote(name)}$/i }
+    #if klass_sym.nil?
+    
+    # Look for a plugin by the config_name
+    klass_sym = base.constants.find { |k| base.const_get(k).config_name == name }
+    klass = base.const_get(klass_sym)
+
+    raise LoadError if klass.nil?
+
+    return klass
+  rescue LoadError => e
+    raise LogStash::PluginLoadingError,
+      I18n.t("logstash.pipeline.plugin-loading-error", :type => type, :name => name, :path => path, :error => e.to_s)
+  end # def load
+end # class LogStash::Plugin

data/lib/logstash/plugin_mixins/aws_config.rb

@@ -0,0 +1,93 @@
+# encoding: utf-8
+require "logstash/config/mixin"
+
+module LogStash::PluginMixins::AwsConfig
+
+  @logger = Cabin::Channel.get(LogStash)
+
+  # This method is called when someone includes this module
+  def self.included(base)
+    # Add these methods to the 'base' given.
+    base.extend(self)
+    base.setup_aws_config
+  end
+
+  US_EAST_1 = "us-east-1"
+  
+  public
+  def setup_aws_config
+    # The AWS Region
+    config :region, :validate => [US_EAST_1, "us-west-1", "us-west-2",
+                                  "eu-west-1", "ap-southeast-1", "ap-southeast-2",
+                                  "ap-northeast-1", "sa-east-1", "us-gov-west-1"], :default => US_EAST_1
+
+    # This plugin uses the AWS SDK and supports several ways to get credentials, which will be tried in this order...   
+    # 1. Static configuration, using `access_key_id` and `secret_access_key` params in logstash plugin config   
+    # 2. External credentials file specified by `aws_credentials_file`   
+    # 3. Environment variables `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY`   
+    # 4. Environment variables `AMAZON_ACCESS_KEY_ID` and `AMAZON_SECRET_ACCESS_KEY`   
+    # 5. IAM Instance Profile (available when running inside EC2)   
+    config :access_key_id, :validate => :string
+
+    # The AWS Secret Access Key
+    config :secret_access_key, :validate => :string
+
+    # Should we require (true) or disable (false) using SSL for communicating with the AWS API   
+    # The AWS SDK for Ruby defaults to SSL so we preserve that
+    config :use_ssl, :validate => :boolean, :default => true
+
+    # URI to proxy server if required
+    config :proxy_uri, :validate => :string
+
+    # Path to YAML file containing a hash of AWS credentials.   
+    # This file will only be loaded if `access_key_id` and
+    # `secret_access_key` aren't set. The contents of the
+    # file should look like this:
+    #
+    #     :access_key_id: "12345"
+    #     :secret_access_key: "54321"
+    #
+    config :aws_credentials_file, :validate => :string
+  end
+
+  public
+  def aws_options_hash
+    if @access_key_id.is_a?(NilClass) ^ @secret_access_key.is_a?(NilClass)
+      @logger.warn("Likely config error: Only one of access_key_id or secret_access_key was provided but not both.")
+    end
+
+    if ((!@access_key_id || !@secret_access_key)) && @aws_credentials_file
+      access_creds = YAML.load_file(@aws_credentials_file)
+
+      @access_key_id = access_creds[:access_key_id]
+      @secret_access_key = access_creds[:secret_access_key]
+    end
+
+    opts = {}
+
+    if (@access_key_id && @secret_access_key)
+      opts[:access_key_id] = @access_key_id
+      opts[:secret_access_key] = @secret_access_key
+    end
+
+    opts[:use_ssl] = @use_ssl
+
+    if (@proxy_uri)
+      opts[:proxy_uri] = @proxy_uri
+    end
+
+    # The AWS SDK for Ruby doesn't know how to make an endpoint hostname from a region
+    # for example us-west-1 -> foosvc.us-west-1.amazonaws.com
+    # So our plugins need to know how to generate their endpoints from a region
+    # Furthermore, they need to know the symbol required to set that value in the AWS SDK
+    # Classes using this module must implement aws_service_endpoint(region:string)
+    # which must return a hash with one key, the aws sdk for ruby config symbol of the service
+    # endpoint, which has a string value of the service endpoint hostname
+    # for example, CloudWatch, { :cloud_watch_endpoint => "monitoring.#{region}.amazonaws.com" }
+    # For a list, see https://github.com/aws/aws-sdk-ruby/blob/master/lib/aws/core/configuration.rb
+    opts.merge!(self.aws_service_endpoint(@region))
+    
+    return opts
+  end # def aws_options_hash
+
+end

data/lib/logstash/program.rb

@@ -0,0 +1,15 @@
+# encoding: utf-8
+
+require "logstash/namespace"
+
+module LogStash::Program
+  public
+  def exit(value)
+    if RUBY_ENGINE == "jruby"
+      # Kernel::exit() in jruby just tosses an exception? Let's actually exit.
+      Java::java.lang.System.exit(value)
+    else
+      Kernel::exit(value)
+    end
+  end # def exit
+end # module LogStash::Program

data/lib/logstash/runner.rb

@@ -0,0 +1,238 @@
+# encoding: utf-8
+
+Encoding.default_external = "UTF-8"
+$START = Time.now
+$DEBUGLIST = (ENV["DEBUG"] || "").split(",")
+
+Thread.abort_on_exception = true
+if ENV["PROFILE_BAD_LOG_CALLS"] || $DEBUGLIST.include?("log")
+  # Set PROFILE_BAD_LOG_CALLS=1 in your environment if you want
+  # to track down logger calls that cause performance problems
+  #
+  # Related research here:
+  #   https://github.com/jordansissel/experiments/tree/master/ruby/logger-string-vs-block
+  #
+  # Basically, the following is wastes tons of effort creating objects that are
+  # never used if the log level hides the log:
+  #
+  #     logger.debug("something happend", :what => Happened)
+  #
+  # This is shown to be 4x faster:
+  #
+  #     logger.debug(...) if logger.debug?
+  #
+  # I originally intended to use RubyParser and SexpProcessor to
+  # process all the logstash ruby code offline, but it was much
+  # faster to write this monkeypatch to warn as things are called.
+  require "cabin/mixins/logger"
+  module Cabin::Mixins::Logger
+    LEVELS.keys.each do |level|
+      m = "original_#{level}".to_sym
+      predicate = "#{level}?".to_sym
+      alias_method m, level
+      define_method(level) do |*args|
+        if !send(predicate)
+          warn("Unconditional log call", :location => caller[0])
+        end
+        send(m, *args)
+      end
+    end
+  end
+end # PROFILE_BAD_LOG_CALLS
+
+if __FILE__ =~ /^(jar:)?file:\//
+  require "logstash/monkeypatches-for-performance"
+end
+require "logstash/monkeypatches-for-bugs"
+require "logstash/monkeypatches-for-debugging"
+require "logstash/namespace"
+require "logstash/program"
+require "i18n" # gem 'i18n'
+I18n.enforce_available_locales = true
+I18n.load_path << File.expand_path(
+  File.join(File.dirname(__FILE__), "../../locales/en.yml")
+)
+
+class LogStash::Runner
+  include LogStash::Program
+
+  def main(args)
+    require "logstash/util"
+    require "stud/trap"
+    require "stud/task"
+    @startup_interruption_trap = Stud::trap("INT") { puts "Interrupted"; exit 0 }
+
+    LogStash::Util::set_thread_name(self.class.name)
+    #$LOAD_PATH << File.join(File.dirname(__FILE__), "..")
+
+    if RUBY_VERSION < "1.9.2"
+      $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+      return 1
+    end
+
+    Stud::untrap("INT", @startup_interruption_trap)
+
+    args = [nil] if args.empty?
+
+    @runners = []
+    while args != nil && !args.empty?
+      args = run(args)
+    end
+
+    status = []
+    @runners.each do |r|
+      #$stderr.puts "Waiting on #{r.wait.inspect}"
+      status << r.wait
+    end
+
+    # Avoid running test/unit's at_exit crap
+    if status.empty? || status.first.nil?
+      exit(0)
+    else
+      exit(status.first)
+    end
+  end # def self.main
+
+  def run(args)
+    command = args.shift
+    commands = {
+      "version" => lambda do
+        require "logstash/agent"
+        agent_args = ["--version"]
+        if args.include?("--verbose") 
+          agent_args << "--verbose"
+        end
+        LogStash::Agent.run($0, agent_args)
+        return []
+      end,
+      "web" => lambda do
+        # Give them kibana.
+        require "logstash/kibana"
+        kibana = LogStash::Kibana::Runner.new
+        @runners << kibana
+        return kibana.run(args)
+      end,
+      "test" => lambda do
+        $LOAD_PATH << File.join(File.dirname(__FILE__), "..", "..", "test")
+        require "logstash/test"
+        test = LogStash::Test.new
+        @runners << test
+        return test.run(args)
+      end,
+      "rspec" => lambda do
+        require "rspec/core/runner"
+        require "rspec"
+        fixedargs = args.collect do |arg|
+          # if the arg ends in .rb or has a "/" in it, assume it's a path.
+          if arg =~ /\.rb$/ || arg =~ /\//
+            # check if it's a file, if not, try inside the jar if we are in it.
+            if !File.exists?(arg) && __FILE__ =~ /file:.*\.jar!\//
+              # Try inside the jar.
+              jar_root = __FILE__.gsub(/!.*/,"!")
+              newpath = File.join(jar_root, arg)
+
+              # Strip leading 'jar:' path (JRUBY_6970)
+              newpath.gsub!(/^jar:/, "")
+              if File.exists?(newpath)
+                # Add the 'spec' dir to the load path so specs can run
+                specpath = File.join(jar_root, "spec")
+                $LOAD_PATH << specpath unless $LOAD_PATH.include?(specpath)
+                next newpath
+              end
+            end
+          end
+          next arg
+        end # args.collect
+
+        # Hack up a runner
+        runner = Class.new do
+          def initialize(args)
+            @args = args
+          end
+          def run
+            @thread = Thread.new do
+              @result = RSpec::Core::Runner.run(@args)
+            end
+          end
+          def wait
+            @thread.join
+            return @result
+          end
+        end
+
+        $LOAD_PATH << File.expand_path("#{File.dirname(__FILE__)}/../../spec")
+        require "test_utils"
+        #p :args => fixedargs
+        rspec = runner.new(fixedargs)
+        rspec.run
+        @runners << rspec
+        return []
+      end,
+      "irb" => lambda do
+        require "irb"
+        IRB.start(__FILE__)
+        return []
+      end,
+      "ruby" => lambda do
+        require(args[0])
+        return []
+      end,
+      "pry" => lambda do
+        require "pry"
+        return binding.pry
+      end,
+      "agent" => lambda do
+        require "logstash/agent"
+        # Hack up a runner
+        agent = LogStash::Agent.new($0)
+        begin
+          agent.parse(args)
+        rescue Clamp::HelpWanted => e
+          puts e.command.help
+        rescue Clamp::UsageError => e
+          # If 'too many arguments' then give the arguments to
+          # the next command. Otherwise it's a real error.
+          raise if e.message != "too many arguments"
+          remaining = agent.remaining_arguments
+        end
+
+        #require "pry"
+        #binding.pry
+        @runners << Stud::Task.new { agent.execute }
+        return remaining
+      end
+    } # commands
+
+    if commands.include?(command)
+      args = commands[command].call
+    else
+      if command.nil?
+        $stderr.puts "No command given"
+      else
+        if !%w(--help -h help).include?(command)
+          # Emit 'no such command' if it's not someone asking for help.
+          $stderr.puts "No such command #{command.inspect}"
+        end
+      end
+      $stderr.puts "Usage: logstash <command> [command args]"
+      $stderr.puts "Run a command with the --help flag to see the arguments."
+      $stderr.puts "For example: logstash agent --help"
+      $stderr.puts
+      # hardcode the available commands to reduce confusion.
+      $stderr.puts "Available commands:"
+      $stderr.puts "  agent - runs the logstash agent"
+      $stderr.puts "  version - emits version info about this logstash"
+      $stderr.puts "  web - runs the logstash web ui (called Kibana)"
+      $stderr.puts "  rspec - runs tests"
+      #$stderr.puts commands.keys.map { |s| "  #{s}" }.join("\n")
+      exit 1
+    end
+
+    return args
+  end # def run
+
+end # class LogStash::Runner
+
+if $0 == __FILE__
+  LogStash::Runner.new.main(ARGV)
+end