logstash-output-vespa_feed 0.7.0 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 49bc383847ec18019e28a453f1d1f173908edbce56659d6146e52160ca696948
-  data.tar.gz: 8e2f0b88b28b9a0293ea3743ef9bf4a24f5e70c17dabf0d5f73a50cfa5afd3b3
+  metadata.gz: da41467fd259e14c680f84a9ab406885326deec8968af3fbaa12f6abcd3c35dc
+  data.tar.gz: 5fbe4d025c77730760f3103d19fd5d61c3249536b4947ea42fa5413fc727c845
 SHA512:
-  metadata.gz: 256154a1e230b1229b838e6c4c260e1323664129906b2b88564bd6b4a25de19c183a049d9868499f63d703a34d9b30f5484cc1bbee50ac6e2cc81383c57b9610
-  data.tar.gz: f27b1f0d06aa8cb5d42fa38074e173ae767088a2108379f51b97b592189cfca960ab967a3cd2078356ea415ca4b7a60d7e4a959261440374701243b6b7502dcc
+  metadata.gz: 233f69ae4a1e2909c800b5c91e4181d9342046aea93c2c7ec423466cb861188b7ea01b1d569063143bc0c3e950f5542718f8586af380eaccb907c2d3c798d8d6
+  data.tar.gz: 885c8572e25dad3b1366cdd2747cbeef834e89bb1e9ced2f444b7bbce740de1db1ab5fa2b3c7e6d930686486570f289862aa487ccf132dfb410384834501dfa8

data/CHANGELOG.md

@@ -1,3 +1,6 @@
+## 0.7.0
+- adds dead letter queue support
+
 ## 0.6.0
 - adds options to remove fields after using them for writing
 

data/README.md

@@ -17,7 +17,7 @@ If you're developing the plugin, you'll want to do something like:
 # run tests
 ./gradlew test
 # install it as a Logstash plugin
-/opt/logstash/bin/logstash-plugin install /path/to/logstash-output-vespa/logstash-output-vespa_feed-0.6.1.gem
+/opt/logstash/bin/logstash-plugin install /path/to/logstash-output-vespa/logstash-output-vespa_feed-0.7.0.gem
 # profit
 /opt/logstash/bin/logstash
 ```
@@ -43,7 +43,7 @@ does the trick.
 
 ## Usage
 
-Logstash config example:
+Some more Logstash config examples can be found [in this blog post](https://blog.vespa.ai/logstash-vespa-tutorials/), but here's one with all the output plugin options:
 
 ```
 # read stuff
@@ -134,6 +134,23 @@ output {
     
     # how many times to retry on transient failures
     max_retries => 10
+
+    # if we we exceed the number of retries or if there are intransient errors,
+    # like field not in the schema, invalid operation, we can send the document to a dead letter queue
+
+    # you'd need to set this to true, default is false. NOTE: this overrides whatever is in logstash.yml
+    enable_dlq => false
+
+    # the path to the dead letter queue. NOTE: the last part of the path is the pipeline ID,
+    # if you want to use the dead letter queue input plugin
+    dlq_path => "data/dead_letter_queue"
+
+    # max dead letter queue size (bytes)
+    max_queue_size => 1073741824
+    # max segment size (i.e. file from the dead letter queue - also in bytes)
+    max_segment_size => 10485760
+    # flush interval (how often to commit the DLQ to disk, in milliseconds)
+    flush_interval => 5000
   }
 }
 ```

data/VERSION

@@ -1 +1 @@
-0.7.0
+0.8.0

data/docs/index.asciidoc

@@ -198,6 +198,47 @@ After this time (seconds), the circuit breaker will be half-open:
 it will ping the endpoint to see if it's back,
 then resume sending requests when it's back.
 
+[id="plugins-{type}s-{plugin}-enable_dlq"]
+===== `enable_dlq`
+
+* Value type is <<boolean,boolean>>
+* Default value is `false`
+
+Enable Dead Letter Queue for this plugin. This overrides the global DLQ setting in logstash.yml.
+In fact, values from logstash.yml are ignored, only those from the plugin configuration matter.
+
+[id="plugins-{type}s-{plugin}-dlq_path"]
+===== `dlq_path`
+
+* Value type is <<string,string>>
+* Default value is `"data/dead_letter_queue"`
+
+Path to the Dead Letter Queue directory. The last part of the path will be the pipeline ID if you want to use the dead letter queue input plugin.
+
+[id="plugins-{type}s-{plugin}-max_queue_size"]
+===== `max_queue_size`
+
+* Value type is <<number,number>>
+* Default value is `1073741824` (1GB)
+
+Maximum size of the Dead Letter Queue in bytes.
+
+[id="plugins-{type}s-{plugin}-max_segment_size"]
+===== `max_segment_size`
+
+* Value type is <<number,number>>
+* Default value is `10485760` (10MB)
+
+Maximum size of each Dead Letter Queue segment file in bytes.
+
+[id="plugins-{type}s-{plugin}-flush_interval"]
+===== `flush_interval`
+
+* Value type is <<number,number>>
+* Default value is `5000`
+
+How often to commit the Dead Letter Queue to disk, in milliseconds.
+
 // The full list of Value Types is here: 
 // https://www.elastic.co/guide/en/logstash/current/configuration-file-structure.html
 

data/lib/logstash-output-vespa_feed_jars.rb

@@ -2,4 +2,4 @@
 # encoding: utf-8
 
 require 'jar_dependencies'
-require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.7.0')
+require_jar('org.logstashplugins', 'logstash-output-vespa_feed', '0.8.0')

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: logstash-output-vespa_feed
 version: !ruby/object:Gem::Version
-  version: 0.7.0
+  version: 0.8.0
 platform: ruby
 authors:
 - Radu Gheorghe
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2025-01-10 00:00:00.000000000 Z
+date: 2025-02-03 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement
@@ -76,7 +76,7 @@ files:
 - lib/logstash-output-vespa_feed_jars.rb
 - lib/logstash/outputs/vespa_feed.rb
 - logstash-output-vespa_feed.gemspec
-- vendor/jar-dependencies/org/logstashplugins/logstash-output-vespa_feed/0.7.0/logstash-output-vespa_feed-0.7.0.jar
+- vendor/jar-dependencies/org/logstashplugins/logstash-output-vespa_feed/0.8.0/logstash-output-vespa_feed-0.8.0.jar
 homepage: https://vespa.ai
 licenses:
 - Apache-2.0