logstash-output-picsv 1.0.5

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 8fe57ac88a14e3ed8becf6f750c9f9708c3aeb38
+  data.tar.gz: bb239e83f2e5c3dd4915ab668fb4299b19d26a54
+SHA512:
+  metadata.gz: 1a0259c8796a3180d6c9c757c19ae7abbd9eeeb27bf75f038f167d245197d6ef72698a3ad2ca6891344156a07aebcb824252a9546e1bf1aa83d11a580feaf5fc
+  data.tar.gz: bdcc0729fed0edd80b5f825a910d49a50b79b91e29c3a53f5985644c1542eb056f903cd7f1041db5a2b93104c4eec5dc2f69a8cb14ad942650196e681893b825

data/.gitignore

@@ -0,0 +1,2 @@
+
+.DS_Store

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+gemspec
+gem "logstash", :github => "elastic/logstash", :branch => "2.4"

data/README.md

@@ -0,0 +1,2 @@
+# logstash-output-picsv
+Logstash output plugin for IBM OA Predictive Insights

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/picsv.rb

@@ -0,0 +1,365 @@
+############################################                                    
+#                                                                               
+# picsv                               
+#                                                                               
+# Logstash mediation output for IOA PI
+#                                                                               
+# Version 030815.1 Robert Mckeown
+# Version 030815.2 Jim Ray                                              
+#                                                                               
+############################################     
+
+require "csv"
+require "logstash/namespace"
+require "logstash/outputs/file"
+require 'java' # for the java data format stuff
+
+# SCACSV - based upon original Logstash CSV output.
+# 
+# Write events to disk in CSV format
+# Write a PI header as the first line in the file
+# Name file per PI convention, based upon first and last timestamps encountered 
+
+class LogStash::Outputs::SCACSV < LogStash::Outputs::File
+
+  config_name "scacsv"
+  milestone 1
+
+  # The field names from the event that should be written to the CSV file.
+  # Fields are written to the CSV in the same order as the array.
+  # If a field does not exist on the event, an empty string will be written.
+  config :fields, :validate => :array, :required => true
+
+
+  # If present, the values here will over-ride the default header
+  # names. Useful if you simply want to provide other names
+  config :header, :validate => :array, :required => false
+  
+  # Options for CSV output. This is passed directly to the Ruby stdlib to\_csv function. 
+  # Full documentation is available here: [http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/index.html].
+  # A typical use case would be to use alternative column or row seperators eg: `csv_options => {"col_sep" => "\t" "row_sep" => "\r\n"}` gives tab seperated data with windows line endings
+  config :csv_options, :validate => :hash, :required => false, :default => Hash.new
+
+  # Name of the output group - used as a prefix in the renamed file
+  config :group, :validate  => :string, :required => true
+  config :max_size, :validate => :number, :default => 0
+  config :file_interval_width, :validate => :string, :default => "" # Allow "" or "hour","day" or "minute"
+  config :flush_interval, :validate => :number, :default => 60
+  config :time_field, :validate => :string, :default => "timestamp"
+#  config :time_format, :validate => :string, :default => "%Y%m%d%H%M%S"
+  config :time_field_format, :validate => :string, :required => true
+  config :timestamp_output_format, :validate => :string, :default => "" # "yyyyMMddHHmmss" # java format
+  config :force_GMT_filenames, :validate => :boolean, :default => false
+
+  config :tz_offset, :validate => :number, :default => 0
+  config :increment_time, :validate => :boolean, :default => false
+
+  public
+  def register
+    super
+    @csv_options = Hash[@csv_options.map{|(k,v)|[k.to_sym, v]}]
+    
+    # variables to hold the start and end times which we'll use to rename the files to
+    @startTime   = "missingStartTime"
+    @endTime     = "missingEndTime"
+    @recordCount = 0
+
+    @lastOutputTime = 0 #data time
+    @flushInterval = @flush_interval.to_i
+
+    @timerThread = Thread.new { flushWatchdog(@flush_interval) }
+
+    @currentOutputIntervalStartTime = 0
+    @fileIntervalWidthSeconds = 0
+    @closeOnIntervalBoundaries = false
+    case @file_interval_width.upcase
+    when "MINUTE"
+      @fileIntervalWidthSeconds = 60
+      @closeOnIntervalBoundaries = true
+    when "FIVE"
+      @fileIntervalWidthSeconds = 300
+      @closeOnIntervalBoundaries = true 
+    when "FIFTEEN"
+      @fileIntervalWidthSeconds = 900
+      @closeOnIntervalBoundaries = true 
+    when "HOUR"
+      @fileIntervalWidthSeconds = 3600
+      @closeOnIntervalBoundaries = true 
+    when "DAY"
+      @fileIntervalWidthSeconds = 86400
+      @closeOnIntervalBoundaries = true
+    else
+      @fileIntervalWidthSeconds = 0 #not used  
+      @closeOnIntervalBoundaries = false
+    end
+
+    @df = nil 
+    if (@time_field_format != "epoch") 
+      @df = java.text.SimpleDateFormat.new(@time_field_format)
+    end
+
+
+  end
+
+  # This thread ensures that we output (close and rename) a file every so often
+  private
+  def flushWatchdog(delay)
+    begin
+      @logger.debug("SCACSVFlushWatchdog - Last output time = " + @lastOutputTime.to_s)
+      while true do
+        @logger.debug("SCACSVFlushWatchdog - Time.now = " + Time.now.to_s + " $lastOutputTime=" + @lastOutputTime.to_s + " delay=" + delay.to_s)
+
+        if ( (Time.now.to_i >= (@lastOutputTime.to_i + delay.to_i)) and (@recordCount > 0)) then
+          @logger.debug("SCACSVFlushWatchdog - closeAndRenameCurrentFile")
+          closeAndRenameCurrentFile
+        end
+        @logger.debug("SCACSVFlushWatchdog - Sleeping")
+        sleep 1
+      end
+    end
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    @logger.debug("in SCACSV receive")
+
+    if (event['SCAWindowMarker']) 
+      # just eat the marker - don't output it
+      # if we had at least one record output, then close the file and move on 
+      if @recordCount >= 1
+        closeAndRenameCurrentFile
+      end
+    else
+
+      # Now see if we need to close file because of a new boundary
+      if @closeOnIntervalBoundaries and @recordCount >= 1 and (@currentOutputIntervalStartTime != snapTimestampToInterval(timestampFromEventAsEpochSeconds(event),@fileIntervalWidthSeconds))
+          closeAndRenameCurrentFile
+      end
+
+      @formattedPath = event.sprintf(@path)
+      fd = open(@formattedPath)
+      @logger.debug("SCACSVreceive - after opening fd=" + fd.to_s)
+
+      if @recordCount == 0
+        # output header on first line - note, need a minimum of one record for sensible output
+        if @header then 
+#         csv_header = @fields.map { |name| name }
+          fd.write(@header.to_csv(@csv_options))
+        else     
+          fd.write(@fields.to_csv(@csv_options))
+        end
+      end
+
+      csv_values = @fields.map {|name| get_value(name, event)}
+      fd.write(csv_values.to_csv(@csv_options))
+
+      flush(fd)
+      close_stale_files
+
+      # remember state
+      @recordCount = @recordCount + 1
+      @lastOutputTime = Time.now
+
+      # capture the earliest - assumption is that records are in order
+      if (@recordCount) == 1 
+        if !@closeOnIntervalBoundaries
+          @startTime = timestampFromEventAsEpochSeconds(event)
+        else
+          @startTime = snapTimestampToInterval(timestampFromEventAsEpochSeconds(event),@fileIntervalWidthSeconds)
+        end
+      end
+
+      # for every record, update endTime - again, assumption is that records are in order
+      if !@closeOnIntervalBoundaries
+        @endTime = timestampFromEventAsEpochSeconds(event)
+      else
+        @endTime = @startTime + @fileIntervalWidthSeconds - 1   # end of interval
+      end
+
+      # remember start of boundary for next time
+      if @closeOnIntervalBoundaries
+         @currentOutputIntervalStartTime = @startTime
+      end
+
+      if ((@max_size > 0) and (@recordCount >= max_size))
+        # Have enough records, close it out
+        closeAndRenameCurrentFile
+      end
+    end  
+
+  end #def receive
+
+  private
+  def timestampAsEpochSeconds(timestamp, dateFormat)
+    if !dateFormat.nil?
+      return dateFormat.parse(timestamp.to_s).getTime/1000 # convert milliSeconds to seconds
+    else
+      #when df not set, we assume epoch SECONDS
+      return timestamp.to_i
+    end
+  end
+
+  private
+  def timestampFromEventAsEpochSeconds(event)
+    timestampAsEpochSeconds(event[@time_field],@df)
+  end
+
+  private
+  def snapTimestampToInterval(timestamp,interval)
+    # timestamp & interval are in seconds
+    intervalStart = (timestamp/ interval) * interval    
+  end
+
+  private
+  def get_value(name, event)
+    val = event[name]
+    case val
+      when Hash
+        return val.to_json
+      else
+        return val
+    end
+  end
+
+  private
+  def epochAsJavaDate( epochTimestamp )
+
+    x = 0
+     if epochTimestamp.to_s.length == 13
+       x = java.util.Date.new(epochTimestamp.to_i)
+     else
+       # should be 10
+       x = java.util.Date.new(epochTimestamp.to_i * 1000)
+     end
+    x
+  end
+
+  def formatOutputTime( timestamp, time_field_format, timestamp_output_format, missingString )
+
+    outputString = ""
+
+    begin
+
+      if timestamp.nil? then
+        @logger.debug("SCACSV " + missingString + " for  #{group}")
+      elsif timestamp_output_format == "epoch" then  
+        outputString = timestamp.to_s  
+      elsif timestamp_output_format == "" then
+        # use time_field format
+        if time_field_format == "epoch" then
+          outputString = timestamp.to_s
+        else
+          df = java.text.SimpleDateFormat.new(time_field_format)
+          if (@force_GMT_filenames) then
+            df.setTimeZone(java.util.TimeZone.getTimeZone("GMT"))
+          end
+          outputString = df.format(epochAsJavaDate(timestamp))
+        end
+      else # explicit java timeformat supplied
+        df = java.text.SimpleDateFormat.new(timestamp_output_format)
+        if (@force_GMT_filenames) then
+          df.setTimeZone(java.util.TimeZone.getTimeZone("GMT"))
+        end
+        outputString = df.format(epochAsJavaDate(timestamp))
+      end
+ 
+    rescue Exception => e
+      @logger.error("Exception determining output file timestamp. " + missingString, :exception => e)
+      outputString = missingString
+    end
+
+    outputString
+
+  end
+
+  def closeAndRenameCurrentFile
+
+    # cloned and changed from the 'file.rb' operator
+    # even though this is in a loop - assumption is that we have one file here for the SCA CSV use
+    @files.each do |path, fd|
+      begin
+        fd.close
+        @files.delete(path) # so it will be forgotten and we can open it up again if needed
+        @logger.debug("closeAndRenameCurrentFile #{path}", :fd => fd)
+
+        # Now the various time adjustments
+
+        begin # determine start&end times
+
+          if (@time_field_format != "epoch")
+            # if not epoch, then we expect java timestamp format
+            # so must convert start/end times
+          #  nStartTime = @df.parse(@startTime)
+          #  nEndTime   = @df.parse(@endTime)
+
+#            @startTime = @df.parse(@startTime).getTime
+#            @endTime   = @df.parse(@endTime).getTime
+
+# All timestamps are in epoch time now
+
+          end
+
+          # Ensure epoch time from here on out
+
+          if (!@startTime.nil?)
+            @startTime = @startTime.to_i + @tz_offset
+          end
+
+          if (!@endTime.nil?)
+            @endTime   = @endTime.to_i + @tz_offset
+            if (@increment_time)
+              # increment is used to ensure that the end-time on the filename is after the last data value
+              @endTime = @endTime.to_i # times are in seconds + 1000 # 1000ms = 1sec
+            end
+          end
+
+          # then do conversion for output
+ 
+         @fileStartTime = formatOutputTime( @startTime, @time_field_format, @timestamp_output_format, "noStartTime" )
+         @fileEndTime   = formatOutputTime( @endTime,   @time_field_format, @timestamp_output_format, "noEndTime" )
+         
+        rescue Exception => e
+          @logger.error("Exception while flushing and closing files - preparing start/end time", :exception => e)
+          raise
+        end
+
+        # timestamps are strings here
+
+        newFilename = "#{group}" + "__" + @fileStartTime + "__" + @fileEndTime + ".csv"
+
+        if newFilename.include? '/'
+          @logger.error("New filename " + newFilename + " cannot contain / characters. Check the timestamp format. / characters stripped from filename")
+          newFilename = newFilename.delete! '/'
+        end
+
+        realdirpath = File.dirname(File.realdirpath("#{path}"))
+        realdirpath = File.dirname(File.realdirpath(path))
+        oldFilename = File.basename(path)
+
+        File.rename(realdirpath + "/" + oldFilename, realdirpath + "/" + newFilename)
+        
+        # reset record count so we'll pick up new start time, and put a header on next file 
+        # when a new record comes in
+        @recordCount = 0
+        @lastOutputTime = Time.now
+
+      rescue Exception => e
+        @logger.error("Exception while flushing and closing files.", :exception => e)
+      end
+    end
+
+  end
+
+  def teardown
+     @logger.debug("SCACSV - Teardown: closing files")
+
+    Thread.kill(@timerThread)
+    closeAndRenameCurrentFile
+
+    finished
+  end
+
+end # class LogStash::Outputs::SCACSV
+

data/logstash-output-picsv.gemspec

@@ -0,0 +1,25 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-picsv'
+  s.version         = "1.0.5"
+  s.licenses = ["Apache-2.0"]
+  s.summary = "Receives a stream of events and outputs files meeting the csv format for IBM Operation Analytics Predictive Insights"
+  # Need to validate 1.5 standalone gemfile compatibility; gemfile exists!
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["ORIG:Robert Mckeown, Update:Jim Ray"]
+  s.email = "raygj@us.ibm.com"
+  s.homepage = "https://github.com/raygj/logstash-output-picsv/README.md"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+  # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.5.0", "< 3.0.0"
+  s.add_runtime_dependency "logstash-codec-plain", "~> 0"
+  s.add_development_dependency "logstash-devutils", "~> 0"
+end

data/spec/outputs/picsv_spec.rb

@@ -0,0 +1,21 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/base" # changed from /example to /base
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::Example do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::Example.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,101 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-picsv
+version: !ruby/object:Gem::Version
+  version: 1.0.5
+platform: ruby
+authors:
+- ORIG:Robert Mckeown, Update:Jim Ray
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2016-09-16 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+  name: logstash-core
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 1.5.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: 3.0.0
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-codec-plain
+  prerelease: false
+  type: :runtime
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+  name: logstash-devutils
+  prerelease: false
+  type: :development
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '0'
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: raygj@us.ibm.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- ".gitignore"
+- Gemfile
+- README.md
+- Rakefile
+- lib/logstash/outputs/picsv.rb
+- logstash-output-picsv.gemspec
+- spec/outputs/picsv_spec.rb
+homepage: https://github.com/raygj/logstash-output-picsv/README.md
+licenses:
+- Apache-2.0
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.4.8
+signing_key:
+specification_version: 4
+summary: Receives a stream of events and outputs files meeting the csv format for IBM Operation Analytics Predictive Insights
+test_files:
+- spec/outputs/picsv_spec.rb