logstash-output-vespa_feed 0.5.1 → 0.6.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ebee2d37b94e57eb0ce3d7c3887fe0966baf6ba239bbd726fa89042e22eee368
-  data.tar.gz: b10386736e36b2310a083f961d4fceefc6037c2290712a9266c1253c6dd1d806
+  metadata.gz: 251a25498ff1530a83bb12e8fe4297b26e06f2e436f9448045eb2b17a6f379e0
+  data.tar.gz: f0ed08c9891c3a05beb615d41d554d46eabefcd06610347878299d0fec5426fc
 SHA512:
-  metadata.gz: 85f2103959d1d000d88e1a11dca26076f4a17e3a8e2d43ff2c178b2c0dd973a83f6738fd7f6cc11a760d45d274994f41b9d9b1acc695bd99f435eda22ca79c61
-  data.tar.gz: fa26fe0a245c96a54e40036c349be73f5ddc62e3d087dbd48129baf4b36667c0c0bfcb3471b6a4c4dc4887ea9b3d4f21a74687c13f7583a31b5ebdfae3981df3
+  metadata.gz: 8aecdcb83a8a4c0bd8d8d73fdf145ef8c68fc99631212fe5c278aaa5a13f071701796ffd11d5fe0e76dc6fb649f68b08311c51f5502c392691c4636458c79a9a
+  data.tar.gz: 47a9ada688f5c2c9addbf694d14fa4f85a2ae8c94d98398a146aced557304a8261fd5391fec0d902562327b276983465359f28c2338ad2ab12b09040d81b7c71

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.6.0
+- adds options to remove fields after using them for writing
+
 ## 0.5.0
 - supports update and remove operations
 

data/README.md

@@ -14,8 +14,10 @@ If you're developing the plugin, you'll want to do something like:
 ```
 # build the gem
 ./gradlew gem
+# run tests
+./gradlew test
 # install it as a Logstash plugin
-/opt/logstash/bin/logstash-plugin install /path/to/logstash-output-vespa/logstash-output-vespa_feed-0.4.0.gem
+/opt/logstash/bin/logstash-plugin install /path/to/logstash-output-vespa/logstash-output-vespa_feed-0.6.0.gem
 # profit
 /opt/logstash/bin/logstash
 ```
@@ -24,6 +26,18 @@ Some more good info about Logstash Java plugins can be found [here](https://www.
 It looks like the JVM options from [here](https://github.com/logstash-plugins/.ci/blob/main/dockerjdk17.env)
 are useful to make JRuby's `bundle install` work.
 
+### Integration tests
+To run integration tests, you'll need to have a Vespa instance running with an app deployed that supports an "id" field. And Logstash installed.
+
+Check out the `integration-test` directory for more information.
+
+```
+cd integration-test
+./run_tests.sh
+```
+
+### Publishing the gem
+
 Note to self: for some reason, `bundle exec rake publish_gem` fails, but `gem push logstash-output-vespa_feed-$VERSION.gem`
 does the trick.
 
@@ -61,7 +75,9 @@ filter {
     columns => ["id", "description", ...]
   }
 
-  # remove fields that we don't need. Here you can do a lot more processing
+  # remove fields we don't need
+  # NOTE: the fields below are added by Logstash by default. You probably *need* this block
+  # otherwise Vespa will reject documents complaining that e.g. @timestamp is an unknown field
   mutate {
     remove_field => ["@timestamp", "@version", "event", "host", "log", "message"]
   }
@@ -99,6 +115,12 @@ output {
     # if the field doesn't exist, we generate a UUID
     id_field => "id"
 
+    # remove fields from the document after using them for writing
+    remove_id => false          # if set to true, remove the ID field after using it
+    remove_namespace => false   # would remove the namespace field (if dynamic)
+    remove_document_type => false # same for document type
+    remove_operation => false   # and operation
+
     # how many HTTP/2 connections to keep open
     max_connections => 1
     # number of streams per connection

data/VERSION

@@ -1 +1 @@
-0.5.1
+0.6.1

data/docs/index.asciidoc

@@ -42,6 +42,10 @@ Writes documents to Vespa.
 | <<plugins-{type}s-{plugin}-operation>> |<<string,string>>|No
 | <<plugins-{type}s-{plugin}-create>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-id_field>> |<<string,string>>|No
+| <<plugins-{type}s-{plugin}-remove_id>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-remove_namespace>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-remove_document_type>> |<<boolean,boolean>>|No
+| <<plugins-{type}s-{plugin}-remove_operation>> |<<boolean,boolean>>|No
 | <<plugins-{type}s-{plugin}-max_retries>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_connections>> |<<number,number>>|No
 | <<plugins-{type}s-{plugin}-max_streams>> |<<number,number>>|No
@@ -77,6 +81,14 @@ Make sure to set the `client_key` as well. The certificate should be in PEM form
 
 Corresponding private key for the `client_cert` certificate. It must not be password-protected, otherwise Logstash will fail to start complaining `Could not find private key in PEM file`.
 
+[id="plugins-{type}s-{plugin}-id_field"]
+===== `id_field`
+
+* Value type is <<string,string>>
+* Default value is `id`
+
+Field to get the document id from. If not present, a UUID will be generated
+
 [id="plugins-{type}s-{plugin}-namespace"]
 ===== `namespace`
 
@@ -104,20 +116,45 @@ Like `namespace`, this can be static or dynamic. So you can use `%{foo}` to use
 
 Operation to perform. Can be `put`, `update`, or `remove`. Dynamic values are supported (e.g. `%{foo}`). If not present, the plugin will use `put`.
 
-[id="plugins-{type}s-{plugin}-create"]
-===== `create`
+
+[id="plugins-{type}s-{plugin}-remove_id"]
+===== `remove_id`
 
 * Value type is <<boolean,boolean>>
+* Default value is `false`
 
-If set to `true`, the plugin will add the `create=true` parameter to the request. It works for `put` and `update` operations, but mostly used for `update` as https://docs.vespa.ai/en/document-v1-api-guide.html#upserts[upserts].
+If set to `true`, removes the ID field from the document after using it for document identification.
 
-[id="plugins-{type}s-{plugin}-id_field"]
-===== `id_field`
+[id="plugins-{type}s-{plugin}-remove_namespace"]
+===== `remove_namespace`
 
-* Value type is <<string,string>>
-* Default value is `id`
+* Value type is <<boolean,boolean>>
+* Default value is `false`
 
-Field to get the document id from. If not present, a UUID will be generated
+When using a dynamic namespace (e.g., `%{my_namespace}`), if set to `true`, removes the field containing the namespace value from the document after using it.
+
+[id="plugins-{type}s-{plugin}-remove_document_type"]
+===== `remove_document_type`
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+When using a dynamic document type (e.g., `%{my_doc_type}`), if set to `true`, removes the field containing the document type value from the document after using it.
+
+[id="plugins-{type}s-{plugin}-remove_operation"]
+===== `remove_operation`
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+When using a dynamic operation (e.g., `%{my_operation}`), if set to `true`, removes the field containing the operation value from the document after using it.
+
+[id="plugins-{type}s-{plugin}-create"]
+===== `create`
+
+* Value type is <<boolean,boolean>>
+
+If set to `true`, the plugin will add the `create=true` parameter to the request. It works for `put` and `update` operations, but mostly used for `update` as https://docs.vespa.ai/en/document-v1-api-guide.html#upserts[upserts].
 
 [id="plugins-{type}s-{plugin}-max_retries"]
 ===== `max_retries`

data/lib/logstash-output-vespa_feed_jars.rb

@@ -2,4 +2,4 @@
 # encoding: utf-8
 
 require 'jar_dependencies'
-require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.5.1')
+require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.6.1')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-vespa_feed
 version: !ruby/object:Gem::Version
-  version: 0.5.1
+  version: 0.6.1
 platform: ruby
 authors:
 - Radu Gheorghe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-12-19 00:00:00.000000000 Z
+date: 2024-12-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -76,7 +76,7 @@ files:
 - lib/logstash-output-vespa_feed_jars.rb
 - lib/logstash/outputs/vespa_feed.rb
 - logstash-output-vespa_feed.gemspec
-- vendor/jar-dependencies/org/logstashplugins/logstash-output-vespa_feed/0.5.1/logstash-output-vespa_feed-0.5.1.jar
+- vendor/jar-dependencies/org/logstashplugins/logstash-output-vespa_feed/0.6.1/logstash-output-vespa_feed-0.6.1.jar
 homepage: https://vespa.ai
 licenses:
 - Apache-2.0