logstash-filter-cache-redis 0.1.0 → 0.2.0

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/config/modules_common.rb

@@ -0,0 +1,71 @@
+# encoding: utf-8
+require "logstash/util/loggable"
+require "logstash/elasticsearch_client"
+require "logstash/modules/importer"
+require "logstash/errors"
+
+module LogStash module Config
+  class ModulesCommon # extracted here for bwc with 5.x
+    include LogStash::Util::Loggable
+
+    def self.pipeline_configs(settings)
+      pipelines = []
+      plugin_modules = LogStash::PLUGIN_REGISTRY.plugins_with_type(:modules)
+
+      modules_array = settings.get("modules.cli").empty? ? settings.get("modules") : settings.get("modules.cli")
+      if modules_array.empty?
+        # no specifed modules
+        return pipelines
+      end
+      logger.debug("Specified modules", :modules_array => modules_array.to_s)
+
+      module_names = modules_array.collect {|module_hash| module_hash["name"]}
+      if module_names.length > module_names.uniq.length
+        duplicate_modules = module_names.group_by(&:to_s).select { |_,v| v.size > 1 }.keys
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-must-be-unique", :duplicate_modules => duplicate_modules)
+      end
+
+      available_module_names = plugin_modules.map(&:module_name)
+      specified_and_available_names = module_names & available_module_names
+
+      if (specified_and_available_names).empty?
+        i18n_opts = {:specified_modules => module_names, :available_modules => available_module_names}
+        raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.modules-unavailable", i18n_opts)
+      end
+
+      specified_and_available_names.each do |module_name|
+        connect_fail_args = {}
+        begin
+          module_hash = modules_array.find {|m| m["name"] == module_name}
+          current_module = plugin_modules.find { |allmodules| allmodules.module_name == module_name }
+
+          alt_name = "module-#{module_name}"
+          pipeline_id = alt_name
+
+          current_module.with_settings(module_hash)
+          esclient = LogStash::ElasticsearchClient.build(module_hash)
+          config_test = settings.get("config.test_and_exit")
+          if esclient.can_connect? || config_test
+            if !config_test
+              current_module.import(LogStash::Modules::Importer.new(esclient))
+            end
+
+            config_string = current_module.config_string
+
+            pipelines << {"pipeline_id" => pipeline_id, "alt_name" => alt_name, "config_string" => config_string, "settings" => settings}
+          else
+            connect_fail_args[:module_name] = module_name
+            connect_fail_args[:hosts] = esclient.host_settings
+          end
+        rescue => e
+          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.parse-failed", :error => e.message)
+        end
+
+        if !connect_fail_args.empty?
+          raise LogStash::ConfigLoadingError, I18n.t("logstash.modules.configuration.elasticsearch_connection_failed", connect_fail_args)
+        end
+      end
+      pipelines
+    end
+  end
+end end

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/elasticsearch_client.rb

@@ -0,0 +1,120 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/logging"
+require "elasticsearch"
+require "elasticsearch/transport/transport/http/manticore"
+
+module LogStash class ElasticsearchClient
+  include LogStash::Util::Loggable
+
+  class Response
+    # duplicated here from Elasticsearch::Transport::Transport::Response
+    # to create a normalised response across different client IMPL
+    attr_reader :status, :body, :headers
+    def initialize(status, body, headers={})
+      @status, @body, @headers = status, body, headers
+      @body = body.force_encoding('UTF-8') if body.respond_to?(:force_encoding)
+    end
+  end
+
+  def self.build(settings)
+    new(RubyClient.new(settings, logger))
+  end
+
+  class RubyClient
+    def initialize(settings, logger)
+      @settings = settings
+      @logger = logger
+      @client_args = client_args
+      @client = Elasticsearch::Client.new(@client_args)
+    end
+
+    def can_connect?
+      begin
+        head(SecureRandom.hex(32).prepend('_'))
+      rescue Elasticsearch::Transport::Transport::Errors::BadRequest
+        true
+      rescue Manticore::SocketException
+        false
+      end
+    end
+
+    def host_settings
+      @client_args[:hosts]
+    end
+
+    def delete(path)
+      begin
+        normalize_response(@client.perform_request('DELETE', path, {}, nil))
+      rescue Exception => e
+        if is_404_error?(e)
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    def put(path, content)
+      normalize_response(@client.perform_request('PUT', path, {}, content))
+    end
+
+    def head(path)
+      begin
+        normalize_response(@client.perform_request('HEAD', path, {}, nil))
+      rescue Exception => e
+        if is_404_error?(e)
+          Response.new(404, "", {})
+        else
+          raise e
+        end
+      end
+    end
+
+    private
+
+    def is_404_error?(error)
+      error.class.to_s =~ /NotFound/ || error.message =~ /Not\s*Found|404/i
+    end
+
+    def normalize_response(response)
+      Response.new(response.status, response.body, response.headers)
+    end
+
+    def client_args
+      {
+        :transport_class => Elasticsearch::Transport::Transport::HTTP::Manticore,
+        :hosts => [*unpack_hosts],
+        # :logger => @logger, # silence the client logging
+      }
+    end
+
+    def unpack_hosts
+      @settings.fetch("var.output.elasticsearch.hosts", "localhost:9200").split(',').map(&:strip)
+    end
+  end
+
+  def initialize(client)
+    @client = client
+  end
+
+  def delete(path)
+    @client.delete(path)
+  end
+
+  def put(path, content)
+    @client.put(path, content)
+  end
+
+  def head(path)
+    @client.head(path)
+  end
+
+  def can_connect?
+    @client.can_connect?
+  end
+
+  def host_settings
+    @client.host_settings
+  end
+end end # class LogStash::ModulesImporter

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/environment.rb

@@ -0,0 +1,183 @@
+# encoding: utf-8
+require "logstash/errors"
+require "logstash/java_integration"
+require "logstash/config/cpu_core_strategy"
+require "logstash/settings"
+require "socket"
+require "stud/temporary"
+
+module LogStash
+  # In the event that we're requiring this file without bootstrap/environment.rb
+  if !defined?(LogStash::Environment::LOGSTASH_HOME)
+    module Environment
+      LOGSTASH_HOME = Stud::Temporary.directory("logstash-home")
+      Dir.mkdir(::File.join(LOGSTASH_HOME, "data"))
+    end
+  end
+
+  [
+            Setting::String.new("node.name", Socket.gethostname),
+    Setting::NullableString.new("path.config", nil, false),
+ Setting::WritableDirectory.new("path.data", ::File.join(LogStash::Environment::LOGSTASH_HOME, "data")),
+    Setting::NullableString.new("config.string", nil, false),
+                    Setting.new("modules.cli", Array, []),
+                    Setting.new("modules", Array, []),
+           Setting::Boolean.new("config.test_and_exit", false),
+           Setting::Boolean.new("config.reload.automatic", false),
+           Setting::Numeric.new("config.reload.interval", 3), # in seconds
+           Setting::Boolean.new("metric.collect", true),
+            Setting::String.new("pipeline.id", "main"),
+            Setting::Boolean.new("pipeline.system", false),
+   Setting::PositiveInteger.new("pipeline.workers", LogStash::Config::CpuCoreStrategy.maximum),
+   Setting::PositiveInteger.new("pipeline.output.workers", 1),
+   Setting::PositiveInteger.new("pipeline.batch.size", 125),
+           Setting::Numeric.new("pipeline.batch.delay", 5), # in milliseconds
+           Setting::Boolean.new("pipeline.unsafe_shutdown", false),
+                    Setting.new("path.plugins", Array, []),
+    Setting::NullableString.new("interactive", nil, false),
+           Setting::Boolean.new("config.debug", false),
+            Setting::String.new("log.level", "info", true, ["fatal", "error", "warn", "debug", "info", "trace"]),
+           Setting::Boolean.new("version", false),
+           Setting::Boolean.new("help", false),
+            Setting::String.new("log.format", "plain", true, ["json", "plain"]),
+            Setting::String.new("http.host", "127.0.0.1"),
+            Setting::PortRange.new("http.port", 9600..9700),
+            Setting::String.new("http.environment", "production"),
+            Setting::String.new("queue.type", "memory", true, ["persisted", "memory", "memory_acked"]),
+            Setting::Boolean.new("queue.drain", false),
+            Setting::Bytes.new("queue.page_capacity", "250mb"),
+            Setting::Bytes.new("queue.max_bytes", "1024mb"),
+            Setting::Numeric.new("queue.max_events", 0), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.acks", 1024), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.writes", 1024), # 0 is unlimited
+            Setting::Numeric.new("queue.checkpoint.interval", 1000), # 0 is no time-based checkpointing
+            Setting::Boolean.new("dead_letter_queue.enable", false),
+            Setting::TimeValue.new("slowlog.threshold.warn", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.info", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.debug", "-1"),
+            Setting::TimeValue.new("slowlog.threshold.trace", "-1")
+  ].each {|setting| SETTINGS.register(setting) }
+
+  # Compute the default queue path based on `path.data`
+  default_queue_file_path = ::File.join(SETTINGS.get("path.data"), "queue")
+  SETTINGS.register Setting::WritableDirectory.new("path.queue", default_queue_file_path)
+  # Compute the default dead_letter_queue path based on `path.data`
+  default_dlq_file_path = ::File.join(SETTINGS.get("path.data"), "dead_letter_queue")
+  SETTINGS.register Setting::WritableDirectory.new("path.dead_letter_queue", default_dlq_file_path)
+
+  SETTINGS.on_post_process do |settings|
+    # If the data path is overridden but the queue path isn't recompute the queue path
+    # We need to do this at this stage because of the weird execution order
+    # our monkey-patched Clamp follows
+    if settings.set?("path.data")
+      if !settings.set?("path.queue")
+        settings.set_value("path.queue", ::File.join(settings.get("path.data"), "queue"))
+      end
+      if !settings.set?("path.dead_letter_queue")
+        settings.set_value("path.dead_letter_queue", ::File.join(settings.get("path.data"), "dead_letter_queue"))
+      end
+    end
+  end
+
+  module Environment
+    extend self
+
+    LOGSTASH_CORE = ::File.expand_path(::File.join(::File.dirname(__FILE__), "..", ".."))
+    LOGSTASH_ENV = (ENV["LS_ENV"] || 'production').to_s.freeze
+
+    LINUX_OS_RE = /linux/
+    WINDOW_OS_RE = /mswin|msys|mingw|cygwin|bccwin|wince|emc/
+    MACOS_OS_RE = /darwin/
+
+    def env
+      LOGSTASH_ENV
+    end
+
+    def production?
+      env.downcase == "production"
+    end
+
+    def development?
+      env.downcase == "development"
+    end
+
+    def test?
+      env.downcase == "test"
+    end
+
+    def runtime_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "runtime-jars")
+    end
+
+    def test_jars_root(dir_name, package)
+      ::File.join(dir_name, package, "test-jars")
+    end
+
+    def load_runtime_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(runtime_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_test_jars!(dir_name="vendor", package="jar-dependencies")
+      load_jars!(::File.join(test_jars_root(dir_name, package), "*.jar"))
+    end
+
+    def load_jars!(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.jruby-required")) unless LogStash::Environment.jruby?
+
+      jar_files = find_jars(pattern)
+      require_jars! jar_files
+    end
+
+    def find_jars(pattern)
+      require 'java'
+      jar_files = Dir.glob(pattern)
+      raise(LogStash::EnvironmentError, I18n.t("logstash.environment.missing-jars", :pattern => pattern)) if jar_files.empty?
+      jar_files
+    end
+
+    def require_jars!(files)
+      files.each do |jar_file|
+        loaded = require jar_file
+        puts("Loaded #{jar_file}") if $DEBUG && loaded
+      end
+    end
+
+    def ruby_bin
+      ENV["USE_RUBY"] == "1" ? "ruby" : File.join("vendor", "jruby", "bin", "jruby")
+    end
+
+    def jruby?
+      @jruby ||= !!(RUBY_PLATFORM == "java")
+    end
+
+    def windows?
+      RbConfig::CONFIG['host_os'] =~ WINDOW_OS_RE
+    end
+
+    def linux?
+      RbConfig::CONFIG['host_os'] =~ LINUX_OS_RE
+    end
+
+    def locales_path(path)
+      return ::File.join(LOGSTASH_CORE, "locales", path)
+    end
+
+    def load_locale!
+      require "i18n"
+      I18n.enforce_available_locales = true
+      I18n.load_path << LogStash::Environment.locales_path("en.yml")
+      I18n.reload!
+      fail "No locale? This is a bug." if I18n.available_locales.empty?
+    end
+
+    # add path for bare/ungemified plugins lookups. the path must be the base path that will include
+    # the dir structure 'logstash/TYPE/NAME.rb' where TYPE is 'inputs' 'filters', 'outputs' or 'codecs'
+    # and NAME is the name of the plugin
+    # @param path [String] plugins path to add
+    def add_plugin_path(path)
+      $LOAD_PATH << path
+    end
+  end
+end
+
+require "logstash/patches"

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/errors.rb

@@ -0,0 +1,13 @@
+# encoding: utf-8
+module LogStash
+  class Error < ::StandardError; end
+  class EnvironmentError < Error; end
+  class ConfigurationError < Error; end
+  class PluginLoadingError < Error; end
+  class ShutdownSignal < StandardError; end
+  class PluginNoVersionError < Error; end
+
+  class Bug < Error; end
+  class ThisMethodWasRemoved < Bug; end
+  class ConfigLoadingError < Error; end
+end

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/execution_context.rb

@@ -0,0 +1,18 @@
+# encoding: utf-8
+require "logstash/util/dead_letter_queue_manager"
+module LogStash
+  class ExecutionContext
+    attr_reader :pipeline, :dlq_writer
+
+    def initialize(pipeline, plugin_id, plugin_type, dlq_writer)
+      @pipeline = pipeline
+      @plugin_id = plugin_id
+      @plugin_type = plugin_type
+      @dlq_writer = LogStash::Util::PluginDeadLetterQueueWriter.new(dlq_writer, @plugin_id, @plugin_type)
+    end
+
+    def pipeline_id
+      @pipeline.pipeline_id
+    end
+  end
+end

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/inputs/base.rb

@@ -0,0 +1,138 @@
+# encoding: utf-8
+require "logstash/namespace"
+require "logstash/event"
+require "logstash/plugin"
+require "logstash/logging"
+require "logstash/config/mixin"
+require "logstash/codecs/base"
+require "logstash/util/decorators"
+
+# This is the base class for Logstash inputs.
+class LogStash::Inputs::Base < LogStash::Plugin
+  include LogStash::Util::Loggable
+  include LogStash::Config::Mixin
+
+  config_name "input"
+
+  # Add a `type` field to all events handled by this input.
+  #
+  # Types are used mainly for filter activation.
+  #
+  # The type is stored as part of the event itself, so you can
+  # also use the type to search for it in Kibana.
+  #
+  # If you try to set a type on an event that already has one (for
+  # example when you send an event from a shipper to an indexer) then
+  # a new input will not override the existing type. A type set at
+  # the shipper stays with that event for its life even
+  # when sent to another Logstash server.
+  config :type, :validate => :string
+
+  config :debug, :validate => :boolean, :default => false, :obsolete => "This setting no longer has any effect. In past releases, it existed, but almost no plugin made use of it."
+
+  config :format, :validate => ["plain", "json", "json_event", "msgpack_event"], :obsolete => "You should use the newer 'codec' setting instead."
+
+  config :charset, :obsolete => "Use the codec setting instead. For example: input { %PLUGIN% { codec => plain { charset => \"UTF-8\" } }"
+
+  config :message_format, :validate => :string, :obsolete => "Setting is no longer valid."
+
+  # The codec used for input data. Input codecs are a convenient method for decoding your data before it enters the input, without needing a separate filter in your Logstash pipeline.
+  config :codec, :validate => :codec, :default => "plain"
+
+  # Add any number of arbitrary tags to your event.
+  #
+  # This can help with processing later.
+  config :tags, :validate => :array
+
+  # Add a field to an event
+  config :add_field, :validate => :hash, :default => {}
+
+  attr_accessor :params
+  attr_accessor :threadable
+
+  def self.plugin_type
+    "input"
+  end
+
+  public
+  def initialize(params={})
+    super
+    @threadable = false
+    @stop_called = Concurrent::AtomicBoolean.new(false)
+    config_init(@params)
+    @tags ||= []
+  end # def initialize
+
+  public
+  def register
+    raise "#{self.class}#register must be overidden"
+  end # def register
+
+  public
+  def tag(newtag)
+    @tags << newtag
+  end # def tag
+
+  public
+  # override stop if you need to do more than do_stop to
+  # enforce the input plugin to return from `run`.
+  # e.g. a tcp plugin might need to close the tcp socket
+  # so blocking read operation aborts
+  def stop
+    # override if necessary
+  end
+
+  public
+  def do_stop
+    @logger.debug("stopping", :plugin => self.class.name)
+    @stop_called.make_true
+    stop
+  end
+
+  # stop? should never be overridden
+  public
+  def stop?
+    @stop_called.value
+  end
+  
+  def clone
+    cloned = super
+    cloned.codec = @codec.clone if @codec
+    cloned
+  end
+
+  def execution_context=(context)
+    super
+    # There is no easy way to propage an instance variable into the codec, because the codec
+    # are created at the class level
+    # TODO(talevy): Codecs should have their own execution_context, for now they will inherit their 
+    #               parent plugin's
+    @codec.execution_context = context
+    context
+  end
+
+  protected
+  def decorate(event)
+    # Only set 'type' if not already set. This is backwards-compatible behavior
+    event.set("type", @type) if @type && !event.include?("type")
+
+    LogStash::Util::Decorators.add_fields(@add_field,event,"inputs/#{self.class.name}")
+    LogStash::Util::Decorators.add_tags(@tags,event,"inputs/#{self.class.name}")
+  end
+
+  protected
+  def fix_streaming_codecs
+    require "logstash/codecs/plain"
+    require "logstash/codecs/line"
+    require "logstash/codecs/json"
+    require "logstash/codecs/json_lines"
+    case @codec
+      when LogStash::Codecs::Plain
+        @logger.info("Automatically switching from #{@codec.class.config_name} to line codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::Line.new("charset" => @codec.charset)
+      when LogStash::Codecs::JSON
+        @logger.info("Automatically switching from #{@codec.class.config_name} to json_lines codec", :plugin => self.class.config_name)
+        @codec = LogStash::Codecs::JSONLines.new("charset" => @codec.charset)
+    end
+  end
+end # class LogStash::Inputs::Base