logstash-filter-cache-redis 0.1.0 → 0.2.0

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/util/wrapped_synchronous_queue.rb

@@ -0,0 +1,319 @@
+# encoding: utf-8
+
+module LogStash; module Util
+  class WrappedSynchronousQueue
+    java_import java.util.concurrent.SynchronousQueue
+    java_import java.util.concurrent.TimeUnit
+
+    def initialize
+      @queue = java.util.concurrent.SynchronousQueue.new
+    end
+
+    # Push an object to the queue if the queue is full
+    # it will block until the object can be added to the queue.
+    #
+    # @param [Object] Object to add to the queue
+    def push(obj)
+      @queue.put(obj)
+    end
+    alias_method(:<<, :push)
+
+    # Offer an object to the queue, wait for the specified amount of time.
+    # If adding to the queue was successful it wil return true, false otherwise.
+    #
+    # @param [Object] Object to add to the queue
+    # @param [Integer] Time in milliseconds to wait before giving up
+    # @return [Boolean] True if adding was successful if not it return false
+    def offer(obj, timeout_ms)
+      @queue.offer(obj, timeout_ms, TimeUnit::MILLISECONDS)
+    end
+
+    # Blocking
+    def take
+      @queue.take
+    end
+
+    # Block for X millis
+    def poll(millis)
+      @queue.poll(millis, TimeUnit::MILLISECONDS)
+    end
+
+    def write_client
+      WriteClient.new(self)
+    end
+
+    def read_client
+      ReadClient.new(self)
+    end
+
+    def close
+      # ignore
+    end
+
+    class ReadClient
+      # We generally only want one thread at a time able to access pop/take/poll operations
+      # from this queue. We also depend on this to be able to block consumers while we snapshot
+      # in-flight buffers
+
+      def initialize(queue, batch_size = 125, wait_for = 250)
+        @queue = queue
+        @mutex = Mutex.new
+        # Note that @inflight_batches as a central mechanism for tracking inflight
+        # batches will fail if we have multiple read clients in the pipeline.
+        @inflight_batches = {}
+
+        # allow the worker thread to report the execution time of the filter + output
+        @inflight_clocks = {}
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def close
+        # noop, compat with acked queue read client
+      end
+
+      def empty?
+        true # synchronous queue is alway empty
+      end
+
+      def set_batch_dimensions(batch_size, wait_for)
+        @batch_size = batch_size
+        @wait_for = wait_for
+      end
+
+      def set_events_metric(metric)
+        @event_metric = metric
+        define_initial_metrics_values(@event_metric)
+      end
+
+      def set_pipeline_metric(metric)
+        @pipeline_metric = metric
+        define_initial_metrics_values(@pipeline_metric)
+      end
+
+      def define_initial_metrics_values(namespaced_metric)
+        namespaced_metric.report_time(:duration_in_millis, 0)
+        namespaced_metric.increment(:filtered, 0)
+        namespaced_metric.increment(:out, 0)
+      end
+
+      def inflight_batches
+        @mutex.lock
+        begin
+          yield(@inflight_batches)
+        ensure
+          @mutex.unlock
+        end
+      end
+
+      def current_inflight_batch
+        @inflight_batches.fetch(Thread.current, [])
+      end
+
+      # create a new empty batch
+      # @return [ReadBatch] a new empty read batch
+      def new_batch
+        ReadBatch.new(@queue, @batch_size, @wait_for)
+      end
+
+      def read_batch
+        batch = new_batch
+        @mutex.lock
+        begin
+          batch.read_next
+        ensure
+          @mutex.unlock
+        end
+        start_metrics(batch)
+        batch
+      end
+
+      def start_metrics(batch)
+        @mutex.lock
+        # there seems to be concurrency issues with metrics, keep it in the mutex
+        begin
+          set_current_thread_inflight_batch(batch)
+          start_clock
+        ensure
+          @mutex.unlock
+        end
+      end
+
+      def set_current_thread_inflight_batch(batch)
+        @inflight_batches[Thread.current] = batch
+      end
+
+      def close_batch(batch)
+        @mutex.lock
+        begin
+          # there seems to be concurrency issues with metrics, keep it in the mutex
+          @inflight_batches.delete(Thread.current)
+          stop_clock(batch)
+        ensure
+          @mutex.unlock
+        end
+      end
+
+      def start_clock
+        @inflight_clocks[Thread.current] = [
+          @event_metric.time(:duration_in_millis),
+          @pipeline_metric.time(:duration_in_millis)
+        ]
+      end
+
+      def stop_clock(batch)
+        unless @inflight_clocks[Thread.current].nil?
+          if batch.size > 0
+            # only stop (which also records) the metrics if the batch is non-empty.
+            # start_clock is now called at empty batch creation and an empty batch could
+            # stay empty all the way down to the close_batch call.
+            @inflight_clocks[Thread.current].each(&:stop)
+          end
+          @inflight_clocks.delete(Thread.current)
+        end
+      end
+
+      def add_filtered_metrics(batch)
+        @event_metric.increment(:filtered, batch.filtered_size)
+        @pipeline_metric.increment(:filtered, batch.filtered_size)
+      end
+
+      def add_output_metrics(batch)
+        @event_metric.increment(:out, batch.filtered_size)
+        @pipeline_metric.increment(:out, batch.filtered_size)
+      end
+    end
+
+    class ReadBatch
+      def initialize(queue, size, wait)
+        @queue = queue
+        @size = size
+        @wait = wait
+
+        @originals = Hash.new
+
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        # @cancelled = Hash.new
+
+        @generated = Hash.new
+        @iterating_temp = Hash.new
+        @iterating = false # Atomic Boolean maybe? Although batches are not shared across threads
+        @acked_batch = nil
+      end
+
+      def read_next
+        @size.times do |t|
+          event = @queue.poll(@wait)
+          return if event.nil? # queue poll timed out
+
+          @originals[event] = true
+        end
+      end
+
+      def merge(event)
+        return if event.nil? || @originals.key?(event)
+        # take care not to cause @generated to change during iteration
+        # @iterating_temp is merged after the iteration
+        if iterating?
+          @iterating_temp[event] = true
+        else
+          # the periodic flush could generate events outside of an each iteration
+          @generated[event] = true
+        end
+      end
+
+      def cancel(event)
+        # TODO: disabled for https://github.com/elastic/logstash/issues/6055 - will have to properly refactor
+        raise("cancel is unsupported")
+        # @cancelled[event] = true
+      end
+
+      def each(&blk)
+        # take care not to cause @originals or @generated to change during iteration
+        @iterating = true
+
+        # below the checks for @cancelled.include?(e) have been replaced by e.cancelled?
+        # TODO: for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+        @originals.each do |e, _|
+          blk.call(e) unless e.cancelled?
+        end
+        @generated.each do |e, _|
+          blk.call(e) unless e.cancelled?
+        end
+        @iterating = false
+        update_generated
+      end
+
+      def size
+        filtered_size
+      end
+
+      def starting_size
+        @originals.size
+      end
+
+      def filtered_size
+        @originals.size + @generated.size
+      end
+
+      def cancelled_size
+      # TODO: disabled for https://github.com/elastic/logstash/issues/6055 = will have to properly refactor
+      raise("cancelled_size is unsupported ")
+        # @cancelled.size
+      end
+
+      private
+
+      def iterating?
+        @iterating
+      end
+
+      def update_generated
+        @generated.update(@iterating_temp)
+        @iterating_temp.clear
+      end
+    end
+
+    class WriteClient
+      def initialize(queue)
+        @queue = queue
+      end
+
+      def get_new_batch
+        WriteBatch.new
+      end
+
+      def push(event)
+        @queue.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def push_batch(batch)
+        batch.each do |event|
+          push(event)
+        end
+      end
+    end
+
+    class WriteBatch
+      def initialize
+        @events = []
+      end
+
+      def size
+        @events.size
+      end
+
+      def push(event)
+        @events.push(event)
+      end
+      alias_method(:<<, :push)
+
+      def each(&blk)
+        @events.each do |e|
+          blk.call(e)
+        end
+      end
+    end
+  end
+end end

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/lib/logstash/version.rb

@@ -0,0 +1,14 @@
+# encoding: utf-8
+
+# The version of the logstash package (not the logstash-core gem version).
+#
+# Note to authors: this should not include dashes because 'gem' barfs if
+# you include a dash in the version string.
+
+# TODO: (colin) the logstash-core gem uses it's own version number in logstash-core/lib/logstash-core/version.rb
+#       there are some dependencies in logstash-core on the LOGSTASH_VERSION constant this is why
+#       the logstash version is currently defined here in logstash-core/lib/logstash/version.rb but
+#       eventually this file should be in the root logstash lib fir and dependencies in logstash-core should be
+#       fixed.
+
+LOGSTASH_VERSION = "5.5.0"

data/vendor/bundle/jruby/1.9/gems/logstash-core-5.5.0-java/locales/en.yml

@@ -0,0 +1,314 @@
+# YAML notes
+#   |- means 'scalar block' useful for formatted text
+#   > means 'scalar block' but it chomps all newlines. Useful
+#     for unformatted text.
+en:
+  oops: |-
+    An unexpected error occurred!
+  logstash:
+    error: >-
+      Error: %{error}
+    environment:
+      jruby-required:  >-
+        JRuby is required
+      missing-jars: >-
+        Could not find jar files under %{pattern}
+    pipeline:
+      worker-error: |-
+        A plugin had an unrecoverable error. Will restart this plugin.
+          Plugin: %{plugin}
+          Error: %{error}
+      worker-error-debug: |-
+        A plugin had an unrecoverable error. Will restart this plugin.
+          Plugin: %{plugin}
+          Error: %{error}
+          Exception: %{exception}
+          Stack: %{stacktrace}
+      plugin-loading-error: >-
+        Couldn't find any %{type} plugin named '%{name}'. Are you
+        sure this is correct? Trying to load the %{name} %{type} plugin
+        resulted in this error: %{error}
+      plugin-type-loading-error: >-
+        Could not find any plugin type named '%{type}'. Check for typos.
+        Valid plugin types are 'input' 'filter' and 'output'
+      output-worker-unsupported: >-
+        %{plugin} output plugin: setting 'workers => %{worker_count}' is not
+        supported by this plugin. I will continue working as if you had not set
+        this setting.
+      output-worker-unsupported-with-message: >-
+        %{plugin} output plugin: setting 'workers => %{worker_count}' is not
+        supported by this plugin. I will continue working as if you had not set
+        this setting. Reason: %{message}
+    plugin:
+      deprecated_milestone: >-
+        %{plugin} plugin is using the 'milestone' method to declare the version
+        of the plugin this method is deprecated in favor of declaring the
+        version inside the gemspec.
+      version:
+        0-9-x:
+         Using version 0.9.x %{type} plugin '%{name}'. This plugin should work but
+         would benefit from use by folks like you. Please let us know if you
+         find bugs or have suggestions on how to improve this plugin.
+        0-1-x: >-
+         Using version 0.1.x %{type} plugin '%{name}'. This plugin isn't well
+         supported by the community and likely has no maintainer.
+    agent:
+      sighup: >-
+        SIGHUP received.
+      sigint: >-
+        SIGINT received. Shutting down the agent.
+      sigterm: >-
+        SIGTERM received. Shutting down the agent.
+      slow_shutdown: |-
+        Received shutdown signal, but pipeline is still waiting for in-flight events
+        to be processed. Sending another ^C will force quit Logstash, but this may cause
+        data loss.
+      forced_sigint: >-
+        SIGINT received. Terminating immediately..
+      non_reloadable_config_reload: >-
+        Unable to reload configuration because it does not support dynamic reloading
+      non_reloadable_config_register: |-
+        Logstash is not able to start since configuration auto reloading was enabled but the configuration contains plugins that don't support it. Quitting...
+    web_api:
+      cant_bind_to_port: |-
+        Logstash tried to bind to port %{port}, but the port is already in use. You can specify a new port by launching logstash with the --http.port option."
+      cant_bind_to_port_in_range: |-
+        Logstash tried to bind to port range %{http_ports}, but all the ports are already in use. You can specify a new port by launching logstash with the --http.port option."
+      hot_threads:
+        title: |-
+          ::: {%{hostname}}
+          Hot threads at %{time}, busiestThreads=%{top_count}:
+        thread_title: |-
+          %{percent_of_cpu_time} % of cpu usage, state: %{thread_state}, thread name: '%{thread_name}'
+      logging:
+        unrecognized_option: |-
+          unrecognized option [%{option}]
+    modules:
+      configuration:
+        parse-failed: |-
+          Failed to parse the module configuration: [%{error}]
+        modules-must-be-unique: >-
+          Only a single instance of any module can be run at a time. Duplicate
+          modules: %{duplicate_modules}
+        modules-invalid-name: >-
+          Invalid module name: %{module_name}
+        modules-variables-malformed: >-
+          Failed to parse module variable %{rawvar}.  Must be in -M
+          "MODULE_NAME.KEY.SUBKEY=VALUE" format
+        modules-unavailable: >-
+          The modules specified are not available yet.
+          Specified modules: %{specified_modules}
+          Available modules: %{available_modules}
+        elasticsearch_connection_failed: >-
+          Failed to import module configurations to Elasticsearch.
+          Module: %{module_name} has hosts: %{hosts}
+
+    runner:
+      short-help: |-
+        usage:
+          bin/logstash -f CONFIG_PATH [-t] [-r] [] [-w COUNT] [-l LOG]
+          bin/logstash --modules MODULE_NAME [-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"] [-t] [-w COUNT] [-l LOG]
+          bin/logstash -e CONFIG_STR [-t] [--log.level fatal|error|warn|info|debug|trace] [-w COUNT] [-l LOG]
+          bin/logstash -i SHELL [--log.level fatal|error|warn|info|debug|trace]
+          bin/logstash -V [--log.level fatal|error|warn|info|debug|trace]
+          bin/logstash --help
+      invalid-configuration: >-
+        The given configuration is invalid. Reason: %{error}
+      missing-configuration: >-
+        No configuration file was specified. Perhaps you forgot to provide
+        the '-f yourlogstash.conf' flag?
+      config-module-exclusive: >-
+        Settings 'path.config' (-f) or 'config.string' (-e) can't be used in conjunction with
+        (--modules) or the "modules:" block in the logstash.yml file.
+      cli-module-override: >-
+        Both command-line and logstash.yml modules configurations detected.
+        Using command-line module configuration and ignoring logstash.yml module
+        configuration.
+      reload-without-config-path: >-
+        Configuration reloading also requires passing a configuration path with '-f yourlogstash.conf'
+      locked-data-path: >-
+        Logstash could not be started because there is already another instance using the configured data directory.  If you wish to run multiple instances, you must change the "path.data" setting.
+      invalid-shell: >-
+        Invalid option for interactive Ruby shell. Use either "irb" or "pry"
+      configtest-flag-information: |-
+        You may be interested in the '--configtest' flag which you can use to validate
+        logstash's configuration before you choose to restart a running system.
+      configuration:
+        obsolete: >-
+          The setting `%{name}` in plugin `%{plugin}` is obsolete and is no
+          longer available. %{extra} If you have any questions about this, you
+          are invited to visit https://discuss.elastic.co/c/logstash and ask.
+        file-not-found: |-
+          No config files found: %{path}. Can you make sure this path is a logstash config file?
+        scheme-not-supported: |-
+          URI scheme not supported: %{path}
+          Either pass a local file path or "file|http://" URI
+        fetch-failed: |-
+          Unable to fetch config from: %{path}
+          Reason: %{message}
+        setting_missing: |-
+          Missing a required setting for the %{plugin} %{type} plugin:
+
+            %{type} {
+              %{plugin} {
+                %{setting} => # SETTING MISSING
+                ...
+              }
+            }
+        setting_invalid: |-
+          Invalid setting for %{plugin} %{type} plugin:
+
+            %{type} {
+              %{plugin} {
+                # This setting must be a %{value_type}
+                # %{note}
+                %{setting} => %{value}
+                ...
+              }
+            }
+        invalid_plugin_settings: >-
+          Something is wrong with your configuration.
+        invalid_plugin_settings_duplicate_keys: |-
+          Duplicate keys found in your configuration: [%{keys}]
+          At line: %{line}, column %{column} (byte %{byte})
+          after %{after}
+        invalid_plugin_register: >-
+          Cannot register %{plugin} %{type} plugin.
+          The error reported is:
+            %{error}
+        plugin_path_missing: >-
+          You specified a plugin path that does not exist: %{path}
+        no_plugins_found: |-
+          Could not find any plugins in "%{path}"
+          I tried to find files matching the following, but found none:
+            %{plugin_glob}
+        log_file_failed: |-
+          Failed to open %{path} for writing: %{error}
+
+          This is often a permissions issue, or the wrong
+          path was specified?
+      flag:
+        # Note: Wrap these at 55 chars so they display nicely when clamp emits
+        # them in an 80-character terminal
+        config: |+
+          Load the logstash config from a specific file
+          or directory.  If a directory is given, all
+          files in that directory will be concatenated
+          in lexicographical order and then parsed as a
+          single config file. You can also specify
+          wildcards (globs) and any matched files will
+          be loaded in the order described above.
+        config-string: |+
+          Use the given string as the configuration
+          data. Same syntax as the config file. If no
+          input is specified, then the following is
+          used as the default input:
+          "%{default_input}"
+          and if no output is specified, then the
+          following is used as the default output:
+          "%{default_output}"
+          If you wish to use both defaults, please use
+          the empty string for the '-e' flag.
+        modules: |+
+          Load Logstash modules.
+          Modules can be defined using multiple instances
+          '--modules module1 --modules module2',
+             or comma-separated syntax
+          '--modules=module1,module2'
+          Cannot be used in conjunction with '-e' or '-f'
+          Use of '--modules' will override modules declared
+          in the 'logstash.yml' file.
+        modules_variable: |+
+          Load variables for module template.
+          Multiple instances of '-M' or
+          '--modules.variable' are supported.
+          Ignored if '--modules' flag is not used.
+          Should be in the format of
+          '-M "MODULE_NAME.var.PLUGIN_TYPE.PLUGIN_NAME.VARIABLE_NAME=VALUE"'
+          as in
+          '-M "example.var.filter.mutate.fieldname=fieldvalue"'
+        configtest: |+
+          Check configuration for valid syntax and then exit.
+        http_host: Web API binding host
+        http_port: Web API http port
+        pipeline-workers: |+
+          Sets the number of pipeline workers to run.
+        pipeline-batch-size: |+
+          Size of batches the pipeline is to work in.
+        pipeline-batch-delay: |+
+          When creating pipeline batches, how long to wait while polling
+          for the next event.
+        path_settings: |+
+          Directory containing logstash.yml file. This can also be
+          set through the LS_SETTINGS_DIR environment variable.
+        path_logs: |+
+            Directory to Write Logstash internal logs to.
+        auto_reload: |+
+          Monitor configuration changes and reload
+          whenever it is changed.
+          NOTE: use SIGHUP to manually reload the config
+        reload_interval: |+
+          How frequently to poll the configuration location
+          for changes, in seconds.
+        log: |+
+          Write logstash internal logs to the given
+          file. Without this flag, logstash will emit
+          logs to standard output.
+        verbosity: |+
+          Increase verbosity of logstash internal logs.
+          Specifying once will show 'informational'
+          logs. Specifying twice will show 'debug'
+          logs. This flag is deprecated. You should use
+          --log-level=info or --log-level=debug instead.
+        version: |+
+          Emit the version of logstash and its friends,
+          then exit.
+        datapath: |+
+          This should point to a writable directory. Logstash
+          will use this directory whenever it needs to store
+          data. Plugins will also have access to this path.
+        pluginpath: |+
+          A path of where to find plugins. This flag
+          can be given multiple times to include
+          multiple paths. Plugins are expected to be
+          in a specific directory hierarchy:
+          'PATH/logstash/TYPE/NAME.rb' where TYPE is
+          'inputs' 'filters', 'outputs' or 'codecs'
+          and NAME is the name of the plugin.
+        log_level: |+
+          Set the log level for logstash. Possible values are:
+            - fatal
+            - error
+            - warn
+            - info
+            - debug
+            - trace
+        unsafe_shutdown: |+
+          Force logstash to exit during shutdown even
+          if there are still inflight events in memory.
+          By default, logstash will refuse to quit until all
+          received events have been pushed to the outputs.
+        rubyshell: |+
+          Drop to shell instead of running as normal.
+          Valid shells are "irb" and "pry"
+        name: |+
+          Specify the name of this logstash instance, if no value is given
+          it will default to the current hostname.
+        agent: |+
+          Specify an alternate agent plugin name.
+        config_debug: |+
+          Print the compiled config ruby code out as a debug log (you must also have --log.level=debug enabled).
+          WARNING: This will include any 'password' options passed to plugin configs as plaintext, and may result
+          in plaintext passwords appearing in your logs!
+        log_format: |+
+          Specify if Logstash should write its own logs in JSON form (one
+          event per line) or in plain text (using Ruby's Object#inspect)
+        debug: |+
+          Set the log level to debug.
+          DEPRECATED: use --log.level=debug instead.
+        verbose: |+
+          Set the log level to info.
+          DEPRECATED: use --log.level=info instead.
+        quiet: |+
+          Set the log level to info.
+          DEPRECATED: use --log.level=quiet instead.