embulk-input-splunk 0.1.2 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: c66c1f6d7c39baf9a075c0b872709c995de3177673587687c5d28392a09775ee
-  data.tar.gz: 2c99db44781145c10a26ee262f93dcf28028d2ed2e9102cd13d9078bc6b77974
+  metadata.gz: bb169aa2ac863553e446617ab350bb93b56f4711ce48c5b96ec0ce47e0e0525e
+  data.tar.gz: d715cadc423c7c1dfb6136b1d1cc15a1030f426cb00ee1374eae2f9059058994
 SHA512:
-  metadata.gz: f60a1f0af55a4eee9d8c93fdbfa1b0ad39c0081997bc235b8e2ffda8973ac1bc9ab24a8ac37f7f9c7669e84d93f9efe1fb1fa01379fd4bf2e435bd8503e68238
-  data.tar.gz: 07701a7b6bd41ed2369df1cac99546dba4840ba1414f6d9a5da4fd3414177cd248a865a38713899648541b0347e17d19b77083cb6b499820274ce9459895dc21
+  metadata.gz: 8cd23ac86966cfc83fd7f7fea8c5d41623639aa2056104c8e4429eb23cf27f62099b1dea2b3167fcd0502d710d0cef3656de0663909ba500e9e35ec72debbc48
+  data.tar.gz: e9507de2e73fdc6324e7de78febcbb85f66166b77b8e3b19687bd965c6d27cfe164920c616ab5a180eaafc93d49043386e366fd79c3d2a091b779e4d7dab53fb

data/README.md

@@ -24,17 +24,25 @@ Note that the time is fetched from Splunk's `_time` field. It is possible to ren
 - **query**: the query you wish to run. It should be prefixed with "search" (string required)
 - **earliest_time**: the earliest time for the splunk search. (string, default: nil, which is unbounded)
 - **latest_time**: the latest time for the splunk search. (string, default: nil, which is unbounded)
+- **incremental**: whether to resume next search from last result time (boolean, default: false)
 
 ### Earliest and latest times
 
 Splunk's required data format is `%Y-%m-%dT%H:%M:%S.%L%:z` which is the required format for `earliest_time` and `latest_time`. 
 In addition, Splunk relative time operations are also accepted, such as -1d@d. For more information, see the [Splunk documentation](https://docs.splunk.com/Documentation/Splunk/7.0.2/SearchReference/SearchTimeModifiers)
 
+### Incremental loads
+
+Incremental support is basic. The logic is:
+
+- always rely on `_time` field in Splunk
+- determine latest `_time` in search
+- use latest `_time` as `earliest_time` in next run
+
 ### Number of returned results
 
 The default Splunk API limits resuts to 100. In this plugin, the limit is not set, so it is possible to generate very large result sets. To limit the number of results, use the `head` or `tail` command in your query.
 
-
 ## Examples
 
 Remember the queries much be prefixed with the search command or they are unlikely not to work.

data/embulk-input-splunk.gemspec

@@ -1,7 +1,7 @@
 
 Gem::Specification.new do |spec|
   spec.name          = "embulk-input-splunk"
-  spec.version       = "0.1.2"
+  spec.version       = "0.1.3"
   spec.authors       = ["Scott Arbeitman"]
   spec.summary       = "Splunk input plugin for Embulk"
   spec.description   = "Loads records from a Splunk query."

data/lib/embulk/input/splunk.rb

@@ -12,6 +12,7 @@ module Embulk
       # Zero means unlimited results. Splunk's default is 100.
       SPLUNK_UNLIMITED_RESULTS = 0
       SPLUNK_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%L%:z"
+      SPLUNK_OUTPUT_FORMAT = "json"
 
       def self.transaction(config, &control)
         # configuration code:
@@ -26,9 +27,13 @@ module Embulk
           
           "earliest_time" => config.param(:earliest_time, :string, default: nil),
           "latest_time" => config.param(:latest_time, :string, default: nil),
-          
-          "incremental" => config.param("incremental", :bool, default: false),          
+
+          "incremental" => config.param("incremental", :bool, default: false),
         }
+        
+        if task["incremental"] && task["latest_time"]
+          Embulk.logger.warn "Incremental is 'true' and latest_time is set. This may have unexpected results."
+        end
 
         columns = [
           Column.new(0, "time", :timestamp),
@@ -60,14 +65,19 @@ module Embulk
           :password => task[:password]
         }
         
-        @service = ::Splunk::connect(splunk_config)
         @query = task["query"]
         @earliest_time, @latest_time = task[:earliest_time], task[:latest_time]
+
+        Embulk.logger.info "Establishing connection to Splunk"
+        @service = ::Splunk::connect(splunk_config)
       end
 
       def run
+        Embulk.logger.info "Running query `#{@query}`"
+        
         stream = @service.create_oneshot(@query,
                                          count: SPLUNK_UNLIMITED_RESULTS,
+                                         output_format: SPLUNK_OUTPUT_FORMAT,
                                          earliest_time: @earliest_time,
                                          latest_time: @latest_time)
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: embulk-input-splunk
 version: !ruby/object:Gem::Version
-  version: 0.1.2
+  version: 0.1.3
 platform: ruby
 authors:
 - Scott Arbeitman