logstash-output-elasticsearch_http 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1ef882b3adf307f0b10e05ce2e0038cf0dd4cb24
+  data.tar.gz: 6a8ca5dad1c65cb9c884afa56df2e74b7265e05a
+SHA512:
+  metadata.gz: 98b122b9caa2a690036ec9cbaa867e1d838bbb6686fdb3d277014d851797e7059350ae5da5173e237bfbbfaf13bd03aec48e198f747194adaaffe4c3bdb34b9b
+  data.tar.gz: fdd7230e50f7f219d0a15a51204908db1969d63d6ae8801c58cf86372bf506aed9d5fdcadb8acc0a05d90cf1275c9607b6c58d36007cafafba93c394dfc251c5

data/.gitignore

@@ -0,0 +1,3 @@
+*.gem
+Gemfile.lock
+.bundle

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/LICENSE

@@ -0,0 +1,13 @@
+Copyright (c) 2012-2015 Elasticsearch <http://www.elasticsearch.org>
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.

data/README.md

@@ -0,0 +1,100 @@
+# Please read!
+## This plugin is being deprecated!
+This plugin exist to provide an easy upgrade path to the new [elasticsearch](http://github.com/logstash-plugins/logstash-output-elasticsearch/) output.
+This means that **new pull requests will not be accepted** here, please direct any issues to the new elasticsearch output.
+
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization.
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+```sh
+bundle exec rspec
+```
+
+The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
+```
+To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example:
+```ruby
+gem "logstash", :github => "elasticsearch/logstash", :ref => "master"
+```
+```ruby
+gem "logstash", :path => "/your/local/logstash"
+```
+
+Then update your dependencies and run your tests:
+
+```sh
+bundle install
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Update Logstash dependencies
+```sh
+rake vendor:gems
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to me that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/elasticsearch_http.rb

@@ -0,0 +1,154 @@
+# encoding: utf-8
+require "logstash/outputs/elasticsearch"
+require "logstash/codecs/base"
+require "logstash/namespace"
+require "forwardable"
+
+# This plugin is deprecated in favor of using elasticsearch output and the http protocol
+# Please update your current configuration!
+#
+# .Example
+# [source,ruby]
+#   elasticsearch {
+#     protocol => 'http'
+#     host => '127.0.0.1'
+#   }
+class LogStash::Outputs::ElasticsearchHTTP < LogStash::Outputs::Base
+  extend Forwardable
+
+  config_name "elasticsearch_http"
+
+  # The index to write events to. This can be dynamic using the %{foo} syntax.
+  # The default value will partition your indices by day so you can more easily
+  # delete old data or only search specific date ranges.
+  config :index, :validate => :string, :default => "logstash-%{+YYYY.MM.dd}"
+
+  # The index type to write events to. Generally you should try to write only
+  # similar events to the same 'type'. String expansion '%{foo}' works here.
+  config :index_type, :validate => :string
+
+  # Starting in Logstash 1.3 (unless you set option "manage_template" to false)
+  # a default mapping template for Elasticsearch will be applied, if you do not
+  # already have one set to match the index pattern defined (default of
+  # "logstash-%{+YYYY.MM.dd}"), minus any variables.  For example, in this case
+  # the template will be applied to all indices starting with logstash-*
+  #
+  # If you have dynamic templating (e.g. creating indices based on field names)
+  # then you should set "manage_template" to false and use the REST API to upload
+  # your templates manually.
+  config :manage_template, :validate => :boolean, :default => true
+
+  # This configuration option defines how the template is named inside Elasticsearch.
+  # Note that if you have used the template management features and subsequently
+  # change this you will need to prune the old template manually, e.g.
+  # curl -XDELETE <http://localhost:9200/_template/OldTemplateName?pretty>
+  # where OldTemplateName is whatever the former setting was.
+  config :template_name, :validate => :string, :default => "logstash"
+
+  # You can set the path to your own template here, if you so desire.
+  # If not the included template will be used.
+  config :template, :validate => :path
+
+  # Overwrite the current template with whatever is configured
+  # in the template and template_name directives.
+  config :template_overwrite, :validate => :boolean, :default => false
+
+  # The hostname or IP address to reach your Elasticsearch server.
+  config :host, :validate => :string, :required => true
+
+  # The port for Elasticsearch HTTP interface to use.
+  config :port, :validate => :number, :default => 9200
+
+  # The HTTP Basic Auth username used to access your elasticsearch server.
+  config :user, :validate => :string, :default => nil
+
+  # The HTTP Basic Auth password used to access your elasticsearch server.
+  config :password, :validate => :password, :default => nil
+
+  # This plugin uses the bulk index api for improved indexing performance.
+  # To make efficient bulk api calls, we will buffer a certain number of
+  # events before flushing that out to Elasticsearch. This setting
+  # controls how many events will be buffered before sending a batch
+  # of events.
+  config :flush_size, :validate => :number, :default => 100
+
+  # The amount of time since last flush before a flush is forced.
+  #
+  # This setting helps ensure slow event rates don't get stuck in Logstash.
+  # For example, if your `flush_size` is 100, and you have received 10 events,
+  # and it has been more than `idle_flush_time` seconds since the last flush,
+  # logstash will flush those 10 events automatically.
+  #
+  # This helps keep both fast and slow log streams moving along in
+  # near-real-time.
+  config :idle_flush_time, :validate => :number, :default => 1
+
+  # The document ID for the index. Useful for overwriting existing entries in
+  # Elasticsearch with the same ID.
+  config :document_id, :validate => :string, :default => nil
+
+  # Set the type of Elasticsearch replication to use. If async
+  # the index request to Elasticsearch to return after the primary
+  # shards have been written. If sync (default), index requests
+  # will wait until the primary and the replica shards have been
+  # written.
+  config :replication, :validate => ['async', 'sync'], :default => 'sync'
+
+  def_delegators :@elasticsearch_output, :teardown, :register, :receive
+
+  def initialize(options = {})
+    super(options)
+
+    @logger = Cabin::Channel.get(LogStash)
+    warning_message = []
+
+    warning_message << "The elasticsearch_http output is replaced by the elasticsearch output and will be removed in a future version of Logstash"
+
+    if options.delete("replication") == "async"
+      warning_message << "Ignoring the async replication option, this option is not recommended and not supported by this plugin"
+    end
+
+    # transform configuration
+    options["host"] = [options['host']]
+    options["protocol"] = "http"
+
+    
+    # Generate a migration configuration for the new elasticsearch output
+    # using the current settings as the base.
+    warning_message << "The following configuration example is based on the options you specified and should work:"
+    warning_message << "elasticsearch {"
+
+    @config.each do |option, value|
+      if display_option?(option, value)
+        warning_message << "#{option} => #{format_value(value)}"
+      end
+    end
+
+    warning_message << "}\n"
+
+    @logger.warn(warning_message.join("\n"))
+    @elasticsearch_output = LogStash::Outputs::ElasticSearch.new(options)
+  end
+
+  private
+  def format_value(value)
+    if value.is_a?(LogStash::Codecs::Base)
+      return "\"#{value.class.to_s.split('::').last.downcase}\""
+    elsif value.is_a?(String)
+      return "\"#{value}\""
+    else
+      return value
+    end
+  end
+
+  def display_option?(option, value)
+    return false if option == 'password' && @config['user'].nil?
+    if !value.nil?
+      if value.is_a?(Array) 
+        return true if value.size > 0
+      elsif value.to_s.size != 0
+        return true
+      end
+    end
+  end
+end

data/logstash-output-elasticsearch_http.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-elasticsearch_http'
+  s.version         = '0.0.1'
+  s.licenses = ['Apache License (2.0)']
+  s.summary = "The elasticsearch_http output is deprecated in favor of the elasticsearch"
+  s.description = "The elasticsearch_http output is deprecated in favor of the elasticsearch output with the protocol set to http"
+  s.authors = ["Elasticsearch"]
+  s.email = 'info@elasticsearch.com'
+  s.homepage = "http://www.elasticsearch.org/guide/en/logstash/current/index.html"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency 'logstash-core', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-output-elasticsearch'
+  s.add_runtime_dependency 'logstash-codec-plain'
+  s.add_development_dependency 'logstash-devutils'
+end

data/spec/outputs/elasticsearch_http_spec.rb

@@ -0,0 +1,84 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elasticsearch_http"
+require_relative "../support/helpers"
+
+describe LogStash::Outputs::ElasticsearchHTTP do
+  let(:options) { { "host" => "127.0.0.1" } }
+
+  context "#register" do
+    subject do
+      LogStash::Outputs::ElasticsearchHTTP.new(options)
+    end
+
+    it 'should register' do
+      expect { subject.register }.to_not raise_error
+    end
+
+    context 'transform and display current config into the new format ' do
+      let(:logger) { MockLogger.new }
+      before do
+        expect(Cabin::Channel).to receive(:get).at_least(1).with(LogStash).and_return(logger)
+      end
+
+      subject do
+        LogStash::Outputs::ElasticsearchHTTP.new(options)
+        logger.content 
+      end
+
+      it "warns that async replication is not supported" do
+        options.merge!( { "replication" => "async" })
+        expect(subject).to match(/Ignoring the async replication option/)
+      end
+
+      # since we are trying to have the same config syntax and
+      # Logstash's config isn't json I have to add this kind of guards
+      it "uses the elasticsearch output" do
+        expect(subject).to match(/elasticsearch {/)
+      end
+
+      it "contains host as array" do
+        expect(subject).to match(/host => \["127.0.0.1"\]/)
+      end
+
+      it "contains the default plain codec" do
+        expect(subject).to match(/codec => \"plain\"/)
+      end
+
+      it "defaults to one worker" do
+        expect(subject).to match(/workers => 1/)
+      end
+
+      it "uses the logstash index" do
+        expect(subject).to match(/index => \"logstash-%{\+YYYY\.MM\.dd}\"/)
+      end
+
+      it "define manages templates" do
+        expect(subject).to match(/manage_template => true/)
+      end
+
+      it "defines template_name" do
+        expect(subject).to match(/template_name => "logstash"/)
+      end
+
+      it "defines template_overwrite" do
+        expect(subject).to match(/template_overwrite => false/)
+      end
+
+      it "defines port" do
+        expect(subject).to match(/port => 9200/)
+      end
+
+      it "defines the flush_size" do
+        expect(subject).to match(/flush_size => 100/)
+      end
+
+      it "defines the idle_flush_time" do
+        expect(subject).to match(/idle_flush_time => 1/)
+      end
+
+      it "defines the protocol to http" do
+        expect(subject).to match(/protocol => "http"/)
+      end
+    end
+  end
+end

data/spec/support/helpers.rb

@@ -0,0 +1,17 @@
+class MockLogger
+  def initialize
+    @content = []
+  end
+
+  def info(*value)
+  end
+  alias_method :debug, :info
+
+  def warn(*value)
+    @content << value.join(' ')
+  end
+
+  def content
+    @content.join("\n")
+  end
+end

metadata

@@ -0,0 +1,118 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-elasticsearch_http
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Elasticsearch
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-03-24 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-output-elasticsearch
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: The elasticsearch_http output is deprecated in favor of the elasticsearch output with the protocol set to http
+email: info@elasticsearch.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE
+- README.md
+- Rakefile
+- lib/logstash/outputs/elasticsearch_http.rb
+- logstash-output-elasticsearch_http.gemspec
+- spec/outputs/elasticsearch_http_spec.rb
+- spec/support/helpers.rb
+homepage: http://www.elasticsearch.org/guide/en/logstash/current/index.html
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: The elasticsearch_http output is deprecated in favor of the elasticsearch
+test_files:
+- spec/outputs/elasticsearch_http_spec.rb
+- spec/support/helpers.rb