embulk-input-splunk 0.1.1 → 0.1.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 97cceb8fac1ece34ef872c15aa8861d735d9307c89450ac8ce343a440c855062
-  data.tar.gz: 896b3b1543169a45aeb94494817b9136064cdb566c72449caf41dca28473a38b
+  metadata.gz: c66c1f6d7c39baf9a075c0b872709c995de3177673587687c5d28392a09775ee
+  data.tar.gz: 2c99db44781145c10a26ee262f93dcf28028d2ed2e9102cd13d9078bc6b77974
 SHA512:
-  metadata.gz: bb553e46a762d8fd09b5655bd1c0621c1d5bf28396c00059cd1a1809cf768d6b70f2dc3bbaf47785899bea94807c73b069bf542bd6b5b47e9ff7494c60a301c0
-  data.tar.gz: '085667e45da0411d52b783def58e0c5187d337113e6913b37651f9159843fea76d44a573f1be69283d04375d9e54ba25a78656e4d25b45f4eba270a9a9ecb558'
+  metadata.gz: f60a1f0af55a4eee9d8c93fdbfa1b0ad39c0081997bc235b8e2ffda8973ac1bc9ab24a8ac37f7f9c7669e84d93f9efe1fb1fa01379fd4bf2e435bd8503e68238
+  data.tar.gz: 07701a7b6bd41ed2369df1cac99546dba4840ba1414f6d9a5da4fd3414177cd248a865a38713899648541b0347e17d19b77083cb6b499820274ce9459895dc21

data/README.md

@@ -2,23 +2,87 @@
 
 A simple plug-in to run a once-off Splunk query and emit the results.
 
+This plugin loads events as two columns: `time` and `event`. `event` is JSON contain the results of your query. You can use filter plugins such as [embulk-filter-expand_json](https://github.com/civitaspo/embulk-filter-expand_json) or [embulk-filter-add_time](https://github.com/treasure-data/embulk-filter-add_time) to convert the json column to typed columns. [Rename filter](http://www.embulk.org/docs/built-in.html#rename-filter-plugin) is also useful to rename the typed columns.
+
+Note that the time is fetched from Splunk's `_time` field. It is possible to rename or reformat this field in the query in a such a way that this plugin will fail or have unexpected results. It is recommended you do not alter the `_time` in the query unless you know what you're doing.
+
 ## Overview
 
-* **Plugin type**: input
-* **Resume supported**: no
-* **Cleanup supported**: no
-* **Guess supported**: no
+- **Plugin type**: input
+- **Resume supported**: no
+- **Cleanup supported**: no
+- **Guess supported**: no
 
 ## Configuration
 
--  **type**: splunk
--  **host**: host of your splunk server (string, required)
--  **username**: splunk username (string, required)
--  **password**: splunk password (string, required)
--  **port**: splunk API port (integer, default: 8089)
--  **query**: the query you wish to run. It should be prefixed with "search" (string required)
+- **type**: splunk
+- **scheme**: HTTP scheme for using the Splunk API (string, default: https)
+- **host**: host of your splunk server (string, required)
+- **username**: splunk username (string, required)
+- **password**: splunk password (string, required)
+- **port**: splunk API port (integer, default: 8089)
+- **query**: the query you wish to run. It should be prefixed with "search" (string required)
+- **earliest_time**: the earliest time for the splunk search. (string, default: nil, which is unbounded)
+- **latest_time**: the latest time for the splunk search. (string, default: nil, which is unbounded)
+
+### Earliest and latest times
+
+Splunk's required data format is `%Y-%m-%dT%H:%M:%S.%L%:z` which is the required format for `earliest_time` and `latest_time`. 
+In addition, Splunk relative time operations are also accepted, such as -1d@d. For more information, see the [Splunk documentation](https://docs.splunk.com/Documentation/Splunk/7.0.2/SearchReference/SearchTimeModifiers)
+
+### Number of returned results
+
+The default Splunk API limits resuts to 100. In this plugin, the limit is not set, so it is possible to generate very large result sets. To limit the number of results, use the `head` or `tail` command in your query.
+
+
+## Examples
+
+Remember the queries much be prefixed with the search command or they are unlikely not to work.
+
+### Unbounded time range
+
+```yaml
+in:
+  type: splunk
+  host: splunk.example.com
+  username: splunk_user
+  password: abc123
+  port: 8089
+  query: search index="main"
+```
+
+### Relative time range
+
+```yaml
+in:
+  type: splunk
+  host: splunk.example.com
+  username: splunk_user
+  password: abc123
+  port: 8089
+  query: search index="main"
+  earliest_time: -1m@m
+```
+
+### Absolute time range
+
+```yaml
+in:
+  type: splunk
+  host: splunk.example.com
+  username: splunk_user
+  password: abc123
+  port: 8089
+  query: search index="main"
+  earliest_time: 2017-01-18T19:23:08.237+11:00
+  latest_time: 2018-01-18T19:23:08.237+11:00
+```
+
+### Complex Searches
+
+For those unfamiliar with YAML, the pipe (|) indicates a multiline value. In Splunk the pipe operator is used for creating multi-step processing.
 
-## Example
+For non-trivial Splunk queries, you should leverage the YAML pipe alongside Splunk pipes for easier to read queries.
 
 ```yaml
 in:
@@ -27,9 +91,13 @@ in:
   username: splunk_user
   password: abc123
   port: 8089
-  query: "search index="main" | head 10"
-out:
-  type: stdout
+  query: |
+    search index="main" |
+    eval foo=bar |
+    where like(bar, "%baz%" |
+    head 100
+  earliest_time: 2017-01-18T19:23:08.237+11:00
+  latest_time: 2018-01-18T19:23:08.237+11:00
 ```
 
 

data/embulk-input-splunk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-splunk"
-  spec.version       = "0.1.1"
+  spec.version       = "0.1.2"
   spec.authors       = ["Scott Arbeitman"]
   spec.summary       = "Splunk input plugin for Embulk"
   spec.description   = "Loads records from a Splunk query."

data/lib/embulk/input/splunk.rb

@@ -16,20 +16,23 @@ module Embulk
       def self.transaction(config, &control)
         # configuration code:
         task = {
+          "scheme" => config.param("scheme", :string, default: "https"),
           "host" => config.param("host", :string),
           "port" => config.param("port", :integer, default: 8089),
           "username" => config.param("username", :string),
           "password" => config.param("password", :string),
+
           "query" => config.param("query", :string),
-          "incremental" => config.param("incremental", :bool, default: false),
-          "time_format" => config.param("time_format", :string, default: SPLUNK_TIME_FORMAT),
           
-          "earliest_time" => config.param(:earliest_time, :string, default: "2010-01-01T00:00:00.000"),
+          "earliest_time" => config.param(:earliest_time, :string, default: nil),
+          "latest_time" => config.param(:latest_time, :string, default: nil),
+          
+          "incremental" => config.param("incremental", :bool, default: false),          
         }
 
         columns = [
           Column.new(0, "time", :timestamp),
-          Column.new(1, "result", :json),
+          Column.new(1, "event", :json),
         ]
 
         resume(task, columns, 1, &control)
@@ -50,7 +53,7 @@ module Embulk
       def init
         # initialization code:
         splunk_config = {
-          :scheme => :https,
+          :scheme => task[:scheme],
           :host => task[:host],
           :port => task[:port],
           :username => task[:username],
@@ -59,20 +62,21 @@ module Embulk
         
         @service = ::Splunk::connect(splunk_config)
         @query = task["query"]
-        @earliest_time = task[:earliest_time]
+        @earliest_time, @latest_time = task[:earliest_time], task[:latest_time]
       end
 
       def run
         stream = @service.create_oneshot(@query,
                                          count: SPLUNK_UNLIMITED_RESULTS,
-                                         earliest_time: @earliest_time)
+                                         earliest_time: @earliest_time,
+                                         latest_time: @latest_time)
 
         reader = ::Splunk::ResultsReader.new(stream)
         
         latest_time_in_results = Time.at(0)
 
         reader.each do |result|
-          event_time = Time.strptime( result["_time"], task[:time_format] )
+          event_time = Time.strptime( result["_time"], SPLUNK_TIME_FORMAT )
           latest_time_in_results = [latest_time_in_results, event_time].max
           
           page_builder.add( [

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-splunk
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Scott Arbeitman