embulk-input-splunk 0.1.3 → 0.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: bb169aa2ac863553e446617ab350bb93b56f4711ce48c5b96ec0ce47e0e0525e
-  data.tar.gz: d715cadc423c7c1dfb6136b1d1cc15a1030f426cb00ee1374eae2f9059058994
+  metadata.gz: daf345bda2bb42c7ae1945b66cd0d70ebdd6c02cb1137c83c9ee34916716e6e4
+  data.tar.gz: 503aabbc4ff9e9c5b0674f4bb910a0b40f1f01d73d53cbf0f89a4702935641ae
 SHA512:
-  metadata.gz: 8cd23ac86966cfc83fd7f7fea8c5d41623639aa2056104c8e4429eb23cf27f62099b1dea2b3167fcd0502d710d0cef3656de0663909ba500e9e35ec72debbc48
-  data.tar.gz: e9507de2e73fdc6324e7de78febcbb85f66166b77b8e3b19687bd965c6d27cfe164920c616ab5a180eaafc93d49043386e366fd79c3d2a091b779e4d7dab53fb
+  metadata.gz: 78f10b454a736fccfb373419c35897c7fcd9fc76970d4116e8101dd536d2e5b9fb108d29a2bebd7c3077d88ef93ef8253dcb3507f77327ddca3e9ab0ddb3de7b
+  data.tar.gz: 398aa35c57d07675a6ad2cf1e0f71b731304d3942644002b28789c898e454c29df171e3ce4ada0d3681434c071b95c0e04412ae48c2e44f5f52809f00de1b556

data/README.md

@@ -2,14 +2,18 @@
 
 A simple plug-in to run a once-off Splunk query and emit the results.
 
-This plugin loads events as two columns: `time` and `event`. `event` is JSON contain the results of your query. You can use filter plugins such as [embulk-filter-expand_json](https://github.com/civitaspo/embulk-filter-expand_json) or [embulk-filter-add_time](https://github.com/treasure-data/embulk-filter-add_time) to convert the json column to typed columns. [Rename filter](http://www.embulk.org/docs/built-in.html#rename-filter-plugin) is also useful to rename the typed columns.
+This plugin uses Splunks `table` command to effeciently and flexibly return results. If you want more flexibility, you can add `_raw` as a table field and then use filter plugins such as [embulk-filter-expand_json](https://github.com/civitaspo/embulk-filter-expand_json) or [embulk-filter-add_time](https://github.com/treasure-data/embulk-filter-add_time) to convert the json column to typed columns. [Rename filter](http://www.embulk.org/docs/built-in.html#rename-filter-plugin) is also useful to rename the typed columns.
 
-Note that the time is fetched from Splunk's `_time` field. It is possible to rename or reformat this field in the query in a such a way that this plugin will fail or have unexpected results. It is recommended you do not alter the `_time` in the query unless you know what you're doing.
+# _time and this plugin
+
+This plugin expects and requires `_time`. If you do not include time in your list of columns, this plugin will automatically add it. It is possible to rename or reformat `_time` in the query in a such a way that this plugin will fail or have unexpected results. It is recommended you do not alter the `_time` in the query unless you know what you're doing. If you need to do something esoteric with `_time`, create another field to work with in your Splunk query.
+
+In addition, as a column we treat `_time` as a String, but only because we couldn't get the plugin to work with timestamps. We'd welcome a pull request to fix this issue.
 
 ## Overview
 
 - **Plugin type**: input
-- **Resume supported**: no
+- **Resume supported**: yes
 - **Cleanup supported**: no
 - **Guess supported**: no
 
@@ -25,6 +29,7 @@ Note that the time is fetched from Splunk's `_time` field. It is possible to ren
 - **earliest_time**: the earliest time for the splunk search. (string, default: nil, which is unbounded)
 - **latest_time**: the latest time for the splunk search. (string, default: nil, which is unbounded)
 - **incremental**: whether to resume next search from last result time (boolean, default: false)
+- **table** array of columns to include in the results (array, default: [])
 
 ### Earliest and latest times
 
@@ -45,7 +50,7 @@ The default Splunk API limits resuts to 100. In this plugin, the limit is not se
 
 ## Examples
 
-Remember the queries much be prefixed with the search command or they are unlikely not to work.
+Remember the queries much be prefixed with the `search` command or they are unlikely not to work. See examples below.
 
 ### Unbounded time range
 
@@ -57,6 +62,9 @@ in:
   password: abc123
   port: 8089
   query: search index="main"
+  table:
+    # We treat time as a string, only because we can't get timestamp + format to work
+    - {name: "_time", type: "string"}
 ```
 
 ### Relative time range
@@ -70,6 +78,10 @@ in:
   port: 8089
   query: search index="main"
   earliest_time: -1m@m
+  table:
+    - {name: "_time", type: "string"}  
+    - {name: "foo", type: "string"}
+    - {name: "bar", type: "long"}
 ```
 
 ### Absolute time range
@@ -84,6 +96,10 @@ in:
   query: search index="main"
   earliest_time: 2017-01-18T19:23:08.237+11:00
   latest_time: 2018-01-18T19:23:08.237+11:00
+  table:
+    - {name: "_time", type: "string"}  
+    - {name: "foo", type: "string"}
+    - {name: "bar", type: "long"}    
 ```
 
 ### Complex Searches
@@ -102,13 +118,15 @@ in:
   query: |
     search index="main" |
     eval foo=bar |
-    where like(bar, "%baz%" |
+    where like(bar, "%baz%") |
     head 100
   earliest_time: 2017-01-18T19:23:08.237+11:00
   latest_time: 2018-01-18T19:23:08.237+11:00
+  table:
+    - {name: "_time", type: "string"}
+    - {name: "foo", type: "string"} # Uses foo from the above query
 ```
 
-
 ## Build
 
 ```

data/embulk-input-splunk.gemspec

@@ -1,7 +1,6 @@
-
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-splunk"
-  spec.version       = "0.1.3"
+  spec.version       = "0.2.0"
   spec.authors       = ["Scott Arbeitman"]
   spec.summary       = "Splunk input plugin for Embulk"
   spec.description   = "Loads records from a Splunk query."

data/lib/embulk/input/splunk.rb

@@ -13,9 +13,11 @@ module Embulk
       SPLUNK_UNLIMITED_RESULTS = 0
       SPLUNK_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%L%:z"
       SPLUNK_OUTPUT_FORMAT = "json"
+      SPLUNK_DEFAULT_TIME_FIELD = "_time"
+      SPLUNK_TIME_FIELD = { "name" => SPLUNK_DEFAULT_TIME_FIELD, "type" => "string" }
 
       def self.transaction(config, &control)
-        # configuration code:
+
         task = {
           "scheme" => config.param("scheme", :string, default: "https"),
           "host" => config.param("host", :string),
@@ -29,16 +31,21 @@ module Embulk
           "latest_time" => config.param(:latest_time, :string, default: nil),
 
           "incremental" => config.param("incremental", :bool, default: false),
+          "table" => config.param("table", :array, default: [])
         }
         
         if task["incremental"] && task["latest_time"]
           Embulk.logger.warn "Incremental is 'true' and latest_time is set. This may have unexpected results."
         end
+        
+        if task["table"].select { |field| field["name"] == "_time" }.empty?
+            Embulk.logger.warn "_time is not included in table. Automatically adding it."
+            task["table"] << SPLUNK_TIME_FIELD
+        end
 
-        columns = [
-          Column.new(0, "time", :timestamp),
-          Column.new(1, "event", :json),
-        ]
+        columns = task["table"].map do |column|
+          Column.new(nil, column["name"], column["type"]&.to_sym || :string, column["format"])
+        end
 
         resume(task, columns, 1, &control)
       end
@@ -48,15 +55,16 @@ module Embulk
         
         next_config_diff = {}
         
-        if task["incremental"]
-          next_config_diff[:earliest_time] = Time.parse( task_reports.first[:latest_time_in_results] ).strftime(SPLUNK_TIME_FORMAT)
-        end
+        latest_time_in_results = task_reports.first[:latest_time_in_results]
         
+        if task["incremental"] && latest_time_in_results.present?
+          next_config_diff[:earliest_time] = latest_time_in_results
+        end
+
         return next_config_diff
       end
 
       def init
-        # initialization code:
         splunk_config = {
           :scheme => task[:scheme],
           :host => task[:host],
@@ -64,14 +72,23 @@ module Embulk
           :username => task[:username],
           :password => task[:password]
         }
-        
-        @query = task["query"]
         @earliest_time, @latest_time = task[:earliest_time], task[:latest_time]
-
+        Embulk.logger.info "Earliest time:  #{@earliest_time} / Latest time: #{@latest_time}"
+        
+        @fields = task["table"].collect { |entry| entry["name"] }
+        Embulk.logger.info "Using fields #{@fields.join', '} in query"
+        
+        @query = build_query( task[:query] )
+        
         Embulk.logger.info "Establishing connection to Splunk"
         @service = ::Splunk::connect(splunk_config)
       end
 
+      def build_query(query)
+        # Append table expression to query. Even if already present in the query, this should do no harm.
+        "#{query} | table #{ @fields.join(", ") } "
+      end
+
       def run
         Embulk.logger.info "Running query `#{@query}`"
         
@@ -83,22 +100,24 @@ module Embulk
 
         reader = ::Splunk::ResultsReader.new(stream)
         
-        latest_time_in_results = Time.at(0)
+        latest_time = nil
 
         reader.each do |result|
-          event_time = Time.strptime( result["_time"], SPLUNK_TIME_FORMAT )
-          latest_time_in_results = [latest_time_in_results, event_time].max
+          #We convert event_time to Ruby time for comparison only. 
+          event_time = Time.strptime( result[SPLUNK_DEFAULT_TIME_FIELD], SPLUNK_TIME_FORMAT )
           
-          page_builder.add( [
-            event_time,
-            result.to_json
-          ] )
+          #We need to keep track of latest time for incremental loads. 
+          # Unfortunately, Splunk was not respecting our sort requests, so we need to do a comparison for each row.
+          latest_time = latest_time.nil? ? event_time : [latest_time, event_time].max
+
+          row = @fields.map { |field| result[ field ] }
+          page_builder.add( row )
         end
         
         page_builder.finish
         
         task_result = {
-          latest_time_in_results: latest_time_in_results
+          latest_time_in_results: latest_time.strftime(SPLUNK_TIME_FORMAT)
         }
 
         return task_result

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-splunk
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Scott Arbeitman
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2018-02-18 00:00:00.000000000 Z
+date: 2018-02-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   requirement: !ruby/object:Gem::Requirement