logstash-input-vespa 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b0c4d47c3bf0a7f5265018a47f773cb938b83b451446d7fc2e77e04f47f9e217
-  data.tar.gz: 9b4fd7081b3c45a8adb0560fbebec89dd07c5c5c08714aef8171c35e198e40e1
+  metadata.gz: cc362062aaf492eb77f990be4f6844b34051ef51c7a47cae6b268e48f0c3012f
+  data.tar.gz: f16fd1a3621feb7efdd4b65f039b94cb40a57434bc2e286e2310d794b00d2eb0
 SHA512:
-  metadata.gz: 5802d28aa5b4949e7506ff44805ea7318d05e6ae9e2f000e89b874f1fbc00fe208f5ae54404b705e6ef40773158ea95ebbbdeb2e21829b9542f3a2ee00fe7a31
-  data.tar.gz: 7907e63f27e29d5a98224ca0090d63dab632b314ce6839ed8265a661a593792d253172db0cd6a06000770ce33219c18b1c13abf7519ea69fdd93952f5e0368e5
+  metadata.gz: 23436b3c39d0259b144a1d82fd7fc970b824b74fd9e1bd90d320fbde5501dae3b42660708acef1e2d16fbec6f955dc945d5dab941dbd15b92eb2cf779ffe5851
+  data.tar.gz: c8d280292ad101c31f8795e7aab8ee3f74c5044e76d688bb354637ad1f6c9aeac5e3cc9e3f3c6472a78da7b02dcabf4c0b104b695961448519b382c38044937a

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.3.0
+added retry+backoff logic
+
 ## 0.2.0
 Added support for mTLS certificates, selector, page_size, backend_concurrency, timeout, from_timestamp, and to_timestamp
 

data/README.md

@@ -20,6 +20,15 @@ export LOGSTASH_SOURCE=1
 bundle exec rspec
 ```
 
+To run integration tests, you'll need to have a Vespa instance running with an app deployed that supports an "id" field. And Logstash installed.
+
+Check out the `integration-test` directory for more information.
+
+```
+cd integration-test
+./run_tests.sh
+```
+
 ## Usage
 
 Minimal Logstash config example:
@@ -50,6 +59,9 @@ input {
     client_cert => "/Users/myuser/.vespa/mytenant.myapp.default/data-plane-public-cert.pem"
     client_key => "/Users/myuser/.vespa/mytenant.myapp.default/data-plane-private-key.pem"
 
+    # as an alternative to mTLS, you can use an authentication token for Vespa Cloud
+    auth_token => "vespa_cloud_TOKEN_GOES_HERE"
+
     # page size
     page_size => 100
 
@@ -62,6 +74,12 @@ input {
     # HTTP request timeout
     timeout => 180
 
+    # maximum retries for failed HTTP requests
+    max_retries => 3
+
+    # delay in seconds for the first retry attempt. We double this delay for each subsequent retry.
+    retry_delay => 1
+
     # lower timestamp bound (microseconds since epoch)
     from_timestamp => 1600000000000000
 
@@ -73,4 +91,6 @@ input {
 output {
   stdout {}
 }
-```
+```
+
+To migrate from one Vespa cluster to another, see [this blog post](https://blog.vespa.ai/logstash-vespa-tutorials/).

data/lib/logstash/inputs/vespa.rb

@@ -37,6 +37,10 @@ class LogStash::Inputs::Vespa < LogStash::Inputs::Base
   # Path to the client key file for mTLS.
   config :client_key, :validate => :path
 
+  # Authentication token for Vespa Cloud
+  # it will be sent as a Bearer token in the Authorization header
+  config :auth_token, :validate => :string
+
   # desired page size for the visit request, i.e. the wantedDocumentCount parameter
   config :page_size, :validate => :number, :default => 100
 
@@ -160,12 +164,19 @@ class LogStash::Inputs::Vespa < LogStash::Inputs::Base
       http = Net::HTTP.new(uri.host, uri.port)
       if uri.scheme == "https"
         http.use_ssl = true
-        http.cert = @cert
-        http.key = @key
-        http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        if @client_cert && @client_key
+          http.cert = @cert
+          http.key = @key
+          http.verify_mode = OpenSSL::SSL::VERIFY_PEER
+        end
       end
 
       request = Net::HTTP::Get.new(uri.request_uri)
+      # Add auth token if provided
+      if @auth_token
+        request['Authorization'] = "Bearer #{@auth_token}"
+      end
+
       http.request(request)
     rescue => e
       retries += 1

data/logstash-input-vespa.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-vespa'
-  s.version         = '0.3.0'
+  s.version         = '0.4.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash input plugin reading from Vespa"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"

data/spec/inputs/vespa_spec.rb

@@ -2,6 +2,8 @@
 require "logstash/devutils/rspec/spec_helper"
 require "logstash/inputs/vespa"
 require "webmock/rspec"
+require 'tempfile'
+require 'openssl'
 
 describe LogStash::Inputs::Vespa do
   let(:config) do
@@ -112,5 +114,175 @@ describe LogStash::Inputs::Vespa do
         expect(a_request(:get, "#{base_uri}?#{uri_params}&continuation=AAAAAA")).to have_been_made.once
       end
     end
+
+    context "when using authentication" do
+      it "adds Bearer token to request headers when auth_token is provided" do
+        config_with_token = config.merge({"auth_token" => "test-token"})
+        plugin = described_class.new(config_with_token)
+        plugin.register
+
+        stub_request(:get, "#{base_uri}?#{uri_params}")
+          .with(headers: { 'Authorization' => 'Bearer test-token' })
+          .to_return(status: 200, body: '{"documents": [], "documentCount": 0}')
+
+        plugin.run(queue)
+        expect(a_request(:get, "#{base_uri}?#{uri_params}")
+          .with(headers: { 'Authorization' => 'Bearer test-token' }))
+          .to have_been_made.once
+      end
+    end
+  end
+
+  describe "#register" do
+    let(:temp_cert) do
+      file = Tempfile.new(['cert', '.pem'])
+      # Create a self-signed certificate for testing
+      key = OpenSSL::PKey::RSA.new(2048)
+      cert = OpenSSL::X509::Certificate.new
+      cert.version = 2
+      cert.serial = 1
+      cert.subject = OpenSSL::X509::Name.parse("/CN=Test")
+      cert.issuer = cert.subject
+      cert.public_key = key.public_key
+      cert.not_before = Time.now
+      cert.not_after = Time.now + 3600
+      
+      # Sign the certificate
+      cert.sign(key, OpenSSL::Digest::SHA256.new)
+      
+      file.write(cert.to_pem)
+      file.close
+      file
+    end
+
+    let(:temp_key) do
+      file = Tempfile.new(['key', '.pem'])
+      # Create a valid RSA key for testing
+      key = OpenSSL::PKey::RSA.new(2048)
+      file.write(key.to_pem)
+      file.close
+      file
+    end
+
+    after do
+      temp_cert.unlink
+      temp_key.unlink
+    end
+
+    it "raises error when only client_cert is provided" do
+      invalid_config = config.merge({"client_cert" => temp_cert.path})
+      plugin = described_class.new(invalid_config)
+      
+      expect { plugin.register }.to raise_error(LogStash::ConfigurationError, 
+        "Both client_cert and client_key must be set, you can't have just one")
+    end
+
+    it "raises error when only client_key is provided" do
+      invalid_config = config.merge({"client_key" => temp_key.path})
+      plugin = described_class.new(invalid_config)
+      
+      expect { plugin.register }.to raise_error(LogStash::ConfigurationError,
+        "Both client_cert and client_key must be set, you can't have just one")
+    end
+
+    it "correctly sets up URI parameters" do
+      full_config = config.merge({
+        "selection" => "true",
+        "from_timestamp" => 1234567890,
+        "to_timestamp" => 2234567890,
+        "page_size" => 50,
+        
+        "backend_concurrency" => 2,
+        "timeout" => 120
+      })
+      
+      plugin = described_class.new(full_config)
+      plugin.register
+      
+      # Access the private @uri_params using send
+      uri_params = plugin.send(:instance_variable_get, :@uri_params)
+      expect(uri_params[:selection]).to eq("true")
+      expect(uri_params[:fromTimestamp]).to eq(1234567890)
+      expect(uri_params[:toTimestamp]).to eq(2234567890)
+      expect(uri_params[:wantedDocumentCount]).to eq(50)
+      expect(uri_params[:concurrency]).to eq(2)
+      expect(uri_params[:timeout]).to eq(120)
+    end
+  end
+
+  describe "#parse_response" do
+    it "handles malformed JSON responses" do
+      response = double("response", :body => "invalid json{")
+      result = plugin.parse_response(response)
+      expect(result).to be_nil
+    end
+
+    it "successfully parses valid JSON responses" do
+      valid_json = {
+        "documents" => [{"id" => "doc1"}],
+        "documentCount" => 1
+      }.to_json
+      response = double("response", :body => valid_json)
+      
+      result = plugin.parse_response(response)
+      expect(result["documentCount"]).to eq(1)
+      expect(result["documents"]).to be_an(Array)
+    end
+  end
+
+  describe "#process_documents" do
+    it "creates events with correct decoration" do
+      documents = [
+        {"id" => "doc1", "fields" => {"field1" => "value1"}},
+        {"id" => "doc2", "fields" => {"field1" => "value2"}}
+      ]
+      
+      # Test that decoration is applied
+      expect(plugin).to receive(:decorate).twice
+      
+      plugin.process_documents(documents, queue)
+      expect(queue.size).to eq(2)
+      
+      event1 = queue.pop
+      expect(event1.get("id")).to eq("doc1")
+      expect(event1.get("fields")["field1"]).to eq("value1")
+      
+      event2 = queue.pop
+      expect(event2.get("id")).to eq("doc2")
+      expect(event2.get("fields")["field1"]).to eq("value2")
+    end
+  end
+
+  describe "#stop" do
+    it "sets stopping flag" do
+      plugin.stop
+      expect(plugin.instance_variable_get(:@stopping)).to be true
+    end
+
+    it "interrupts running visit operation" do
+      request_made = Queue.new  # Use a Queue for thread synchronization
+
+      # Setup a response that would normally continue
+      stub_request(:get, "#{base_uri}?#{uri_params}")
+        .to_return(status: 200, body: {
+          documents: [{"id" => "doc1"}],
+          documentCount: 1,
+          continuation: "token"
+        }.to_json)
+        .with { |req| request_made.push(true); true }  # Signal when request is made
+
+      # Run in a separate thread
+      thread = Thread.new { plugin.run(queue) }
+      
+      # Wait for the first request to be made
+      request_made.pop
+      
+      # Now we know the first request has been made, stop the plugin
+      plugin.stop
+      thread.join
+      
+      # Should only make one request despite having a continuation token
+      expect(a_request(:get, "#{base_uri}?#{uri_params}")).to have_been_made.once
+    end
   end
 end 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-vespa
 version: !ruby/object:Gem::Version
-  version: 0.3.0
+  version: 0.4.0
 platform: ruby
 authors:
 - Radu Gheorghe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-19 00:00:00.000000000 Z
+date: 2025-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement