logstash-output-elastic_app_search 1.0.0 → 1.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bbb2ae52ad36bc6b5ebb7234c062f49aaf31f55e9be0da33b3c653307dd385cb
-  data.tar.gz: 6bb21f904b708bb76b52ffb9cb857655e149b78db2f6057aea18dcab062fcb33
+  metadata.gz: fa8fd8e94cc7f5466fd464fd69b0befe6f2232162504a998de0a84b76a1595b5
+  data.tar.gz: 5fd33c920d8f6331d6205cb4edfd602b5b2797cd0905e1ea7c149d2a249e2cfa
 SHA512:
-  metadata.gz: 498ee5a0621c2b42b5fb781daf75561f5c289333092c64f9a6563f8e419c9ce6143ff737b9bd3143923946c73e164b9192b338bc4f5bcdd30ff9ef2bb5666957
-  data.tar.gz: ef8154b9cf148657a72f8baaad838bb66bf56c89cc5ef369ea3b7ceffdd3a7da18768ccdbeea4a90b5f0f597af178731c59920373ba96292e7808111811e929d
+  metadata.gz: dc1827cb1a0787a28486b8abd1aa09572e719985898b6db1bf803bebf225f7460067dfc94b4c64bfbd4f00d8d14ba47baf4693cad8629b12c6d9b82d7be989dd
+  data.tar.gz: 11833676a17f98e624f31b9735f4238b2a0fdf3ff118c4dc9b35854316f4a531a3a65d46fab61a53b5d893babc5ac884945fa04570c0dfd61141cc99564b7639

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 1.1.0
+  - Switched AppSearch client library from Java to Ruby [#12](https://github.com/logstash-plugins/logstash-output-elastic_app_search/issues/12)
+  - Covered with integration tests and dockerized local AppSearch server instance.
+
 ## 1.0.0
   - Added support for On Premise installations of Elastic App Search
   - Updated java client to 0.4.1

data/Gemfile

@@ -9,3 +9,5 @@ if Dir.exist?(logstash_path) && use_logstash_source
   gem 'logstash-core', :path => "#{logstash_path}/logstash-core"
   gem 'logstash-core-plugin-api', :path => "#{logstash_path}/logstash-core-plugin-api"
 end
+
+gem 'elastic-app-search', '~>7.8.0'

data/LICENSE

@@ -1,11 +1,202 @@
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
 
-    http://www.apache.org/licenses/LICENSE-2.0
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2020 Elastic and contributors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

data/README.md

@@ -43,6 +43,22 @@ bundle install
 bundle exec rspec
 ```
 
+#### CI in local machine
+The plugins comes with a `.ci` folder that contains Docker configurations to run on the CI, however to run it locally
+you have to a local checkout of the [CI's infrastructure project](https://github.com/logstash-plugins/.ci/) and symlink 
+some files from that into the local clone's `.ci`. Supposing you have the CI project in <CI> the steps from local clone 
+ot the plugin, are:
+- `cd .ci`
+- `ln -s <CI>/.ci/docker-setup.sh docker-setup.sh`
+- `ln -s <CI>/.ci/docker-run.sh docker-run.sh`
+- `ln -s <CI>/.ci/docker-compose.yml docker-compose.yml`
+- `ln -s <CI>/.ci/docker.env docker.env`
+
+Now export the `ELASTIC_STACK_VERSION` and setup and run docker bash scripts:
+- `export ELASTIC_STACK_VERSION=7.7.0`
+- `.ci/docker-setup.sh` 
+- `.ci/docker-run.sh` 
+
 ### 2. Running your unpublished Plugin in Logstash
 
 #### 2.1 Run in a local Logstash clone

data/lib/logstash/outputs/elastic_app_search.rb

@@ -1,7 +1,6 @@
 # encoding: utf-8
 require "logstash/outputs/base"
-require 'logstash-output-elastic_app_search_jars.rb'
-java_import com.swiftype.appsearch.Client
+require "elastic-app-search"
 
 class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
   config_name "elastic_app_search"
@@ -23,9 +22,9 @@ class LogStash::Outputs::ElasticAppSearch < LogStash::Outputs::Base
     elsif @host && path_is_set?  # because path has a default value we need extra work to if the user set it
       raise ::LogStash::ConfigurationError.new("The setting \"path\" is not compatible with \"host\". Use \"path\" only with \"url\".")
     elsif @host
-      @client = Client.new(@host, @api_key.value)
+      @client = Elastic::AppSearch::Client.new(:host_identifier => @host, :api_key => @api_key.value)
     elsif @url
-      @client = Client.new(nil, @api_key.value, @url + @path)
+      @client = Elastic::AppSearch::Client.new(:api_endpoint => @url + @path, :api_key => @api_key.value)
     end
     check_connection!
   rescue => e

data/logstash-output-elastic_app_search.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name          = 'logstash-output-elastic_app_search'
-  s.version       = '1.0.0'
+  s.version       = '1.1.0'
   s.licenses      = ['Apache-2.0']
   s.summary       = 'Index data to Elastic App Search'
   s.description   = 'This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program'

data/spec/integration/outputs/index_spec.rb

@@ -0,0 +1,103 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/elastic_app_search"
+require "logstash/codecs/plain"
+require "logstash/event"
+require "json"
+
+describe "indexing against running AppSearch", :integration => true do
+
+  let(:engine_name) do
+    (0...10).map { ('a'..'z').to_a[rand(26)] }.join
+  end
+
+  let(:config) do
+    {
+      "api_key" => ENV['APPSEARCH_PRIVATE_KEY'],
+      "engine" => engine_name,
+      "url" => "http://appsearch:3002"
+    }
+  end
+
+  subject(:app_search_output) { LogStash::Outputs::ElasticAppSearch.new(config) }
+
+  before(:each) do
+    create_engine(engine_name, "http://appsearch:3002", ENV['APPSEARCH_PRIVATE_KEY'])
+  end
+
+  private
+  def create_engine(engine_name, host, api_key)
+    url = host + "/api/as/v1/engines"
+    resp = Faraday.post(url, "{\"name\": \"#{engine_name}\"}",
+      "Content-Type" => "application/json",
+      "Authorization" => "Bearer " + api_key)
+    expect(resp.status).to eq(200)
+  end
+
+  describe "search and private keys are configured" do
+    let(:api_key_settings) do
+      {
+        :private => ENV['APPSEARCH_PRIVATE_KEY'],
+        :search => ENV['APPSEARCH_SEARCH_KEY']
+      }
+    end
+
+    it "setup apikeys" do
+      expect(api_key_settings[:private]).to start_with("private-")
+      expect(api_key_settings[:search]).to start_with("search-")
+    end
+  end
+
+  describe "indexing" do
+
+    before do
+      app_search_output.register
+    end
+
+    describe "single event" do
+      let(:event) { LogStash::Event.new("message" => "an event to index") }
+
+      it "should be indexed" do
+        app_search_output.multi_receive([event])
+
+        results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+          attempt_response = execute_search_call(engine_name)
+          expect(attempt_response.status).to eq(200)
+          parsed_resp = JSON.parse(attempt_response.body)
+          expect(parsed_resp.dig("meta", "page", "total_pages")).to eq(1)
+          parsed_resp["results"]
+        end
+        expect(results.first.dig("message", "raw")).to eq "an event to index"
+      end
+    end
+
+    private
+    def execute_search_call(engine_name)
+      url = config["url"] + "/api/as/v1/engines/#{engine_name}/search"
+      resp = Faraday.post(url, '{"query": "event"}',
+            "Content-Type" => "application/json",
+            "Authorization" => "Bearer " + config["api_key"])
+    end
+
+    describe "multiple events" do
+      let(:events) { generate_events(200) } #2 times the slice size used to batch
+
+      it "all should be indexed" do
+        app_search_output.multi_receive(events)
+        results = Stud.try(20.times, RSpec::Expectations::ExpectationNotMetError) do
+          attempt_response = execute_search_call(engine_name)
+          expect(attempt_response.status).to eq(200)
+          parsed_resp = JSON.parse(attempt_response.body)
+          expect(parsed_resp.dig("meta", "page", "total_results")).to eq(200)
+          parsed_resp["results"]
+        end
+        expect(results.first.dig("message", "raw")).to start_with("an event to index")
+      end
+    end
+
+    private
+    def generate_events(num_events)
+      (1..num_events).map { |i| LogStash::Event.new("message" => "an event to index #{i}")}
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-elastic_app_search
 version: !ruby/object:Gem::Version
-  version: 1.0.0
+  version: 1.1.0
 platform: ruby
 authors:
 - Joao Duarte
@@ -9,7 +9,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2019-06-12 00:00:00.000000000 Z
+date: 2020-07-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -81,11 +81,10 @@ files:
 - Gemfile
 - LICENSE
 - README.md
-- lib/logstash-output-elastic_app_search_jars.rb
 - lib/logstash/outputs/elastic_app_search.rb
 - logstash-output-elastic_app_search.gemspec
+- spec/integration/outputs/index_spec.rb
 - spec/outputs/appsearch_spec.rb
-- vendor/swiftype-app-search-0.4.1-all.jar
 homepage: https://elastic.co
 licenses:
 - Apache-2.0
@@ -113,4 +112,5 @@ signing_key:
 specification_version: 4
 summary: Index data to Elastic App Search
 test_files:
+- spec/integration/outputs/index_spec.rb
 - spec/outputs/appsearch_spec.rb

data/lib/logstash-output-elastic_app_search_jars.rb

@@ -1 +0,0 @@
-require_relative '../vendor/swiftype-app-search-0.4.1-all.jar'