logstash-output-scacsv 1.0.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 02e5d4b3c9ceaa6e7968a2ec2e3563e1b9bb2119
+  data.tar.gz: a979b0ca6d5bb260769f9f5e83d5e315e7e04728
+SHA512:
+  metadata.gz: f0b09ab0d67f2e4cee37eb3bd34c3e75c06ebf16e60ea346d511dccc90d402dd73e4b4d713d9dbb990ff54f52bbce0e28a8bc3d66b95fe7329377b090dcdd677
+  data.tar.gz: 9ba0172228e956cb5a0c7cb71a4851c6b8424f9d3d631fd2a5468e02c776fc0346f404f76fada3f6a2178c470381f107873d660792ffce9d8003d3d5f80ada29

data/Gemfile

@@ -0,0 +1,2 @@
+source 'https://rubygems.org'
+gemspec

data/README.md

@@ -0,0 +1,86 @@
+# Logstash Plugin
+
+This is a plugin for [Logstash](https://github.com/elasticsearch/logstash).
+
+It is fully free and fully open source. The license is Apache 2.0, meaning you are pretty much free to use it however you want in whatever way.
+
+## Documentation
+
+Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
+
+- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
+- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
+
+## Need Help?
+
+Need help? Try #logstash on freenode IRC or the https://discuss.elastic.co/c/logstash discussion forum.
+
+## Developing
+
+### 1. Plugin Developement and Testing
+
+#### Code
+- To get started, you'll need JRuby with the Bundler gem installed.
+
+- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization. We also provide [example plugins](https://github.com/logstash-plugins?query=example).
+
+- Install dependencies
+```sh
+bundle install
+```
+
+#### Test
+
+- Update your dependencies
+
+```sh
+bundle install
+```
+
+- Run tests
+
+```sh
+bundle exec rspec
+```
+
+### 2. Running your unpublished Plugin in Logstash
+
+#### 2.1 Run in a local Logstash clone
+
+- Edit Logstash `Gemfile` and add the local plugin path, for example:
+```ruby
+gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
+```
+- Install plugin
+```sh
+bin/plugin install --no-verify
+```
+- Run Logstash with your plugin
+```sh
+bin/logstash -e 'filter {awesome {}}'
+```
+At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
+
+#### 2.2 Run in an installed Logstash
+
+You can use the same **2.1** method to run your plugin in an installed Logstash by editing its `Gemfile` and pointing the `:path` to your local plugin development directory or you can build the gem and install it using:
+
+- Build your plugin gem
+```sh
+gem build logstash-filter-awesome.gemspec
+```
+- Install the plugin from the Logstash home
+```sh
+bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
+```
+- Start Logstash and proceed to test the plugin
+
+## Contributing
+
+All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
+
+Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
+
+It is more important to the community that you are able to contribute.
+
+For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/Rakefile

@@ -0,0 +1 @@
+require "logstash/devutils/rake"

data/lib/logstash/outputs/scacsv.rb

@@ -0,0 +1,292 @@
+############################################                                    
+#                                                                               
+# Scacsv                               
+#                                                                               
+# Logstash mediation output for SCAPI
+#                                                                               
+# Version 160215.1 Robert Mckeown                                               
+#                                                                               
+############################################     
+
+require "csv"
+require "logstash/namespace"
+require "logstash/outputs/file"
+require 'java' # for the java data format stuff
+
+# SCACSV - based upon original Logstash CSV output.
+# 
+# Write events to disk in CSV format
+# Write a PI header as the first line in the file
+# Name file per PI convention, based upon first and last timestamps encountered 
+
+class LogStash::Outputs::SCACSV < LogStash::Outputs::File
+
+  config_name "scacsv"
+  milestone 1
+
+  # The field names from the event that should be written to the CSV file.
+  # Fields are written to the CSV in the same order as the array.
+  # If a field does not exist on the event, an empty string will be written.
+  config :fields, :validate => :array, :required => true
+
+
+  # If present, the values here will over-ride the default header
+  # names. Useful if you simply want to provide other names
+  config :header, :validate => :array, :required => false
+  
+  # Options for CSV output. This is passed directly to the Ruby stdlib to\_csv function. 
+  # Full documentation is available here: [http://ruby-doc.org/stdlib-2.0.0/libdoc/csv/rdoc/index.html].
+  # A typical use case would be to use alternative column or row seperators eg: `csv_options => {"col_sep" => "\t" "row_sep" => "\r\n"}` gives tab seperated data with windows line endings
+  config :csv_options, :validate => :hash, :required => false, :default => Hash.new
+
+  # Name of the output group - used as a prefix in the renamed file
+  config :group, :validate  => :string, :required => true
+  config :max_size, :validate => :number, :default => 0
+  config :flush_interval, :validate => :number, :default => 60
+  config :time_field, :validate => :string, :default => "timestamp"
+#  config :time_format, :validate => :string, :default => "%Y%m%d%H%M%S"
+  config :time_field_format, :validate => :string, :required => true
+  config :timestamp_output_format, :validate => :string, :default => "" # "yyyyMMddHHmmss" # java format
+
+
+
+  config :tz_offset, :validate => :number, :default => 0
+  config :increment_time, :validate => :boolean, :default => false
+
+  public
+  def register
+    super
+    @csv_options = Hash[@csv_options.map{|(k,v)|[k.to_sym, v]}]
+    
+    # variables to hold the start and end times which we'll use to rename the files to
+    @startTime   = "missingStartTime"
+    @endTime     = "missingEndTime"
+    @recordCount = 0
+
+    @lastOutputTime = 0
+    @flushInterval = @flush_interval.to_i
+
+    @timerThread = Thread.new { flushWatchdog(@flush_interval) }
+
+  end
+
+  # This thread ensures that we output (close and rename) a file every so often
+  private
+  def flushWatchdog(delay)
+    begin
+      @logger.debug("SCACSVFlushWatchdog - Last output time = " + @lastOutputTime.to_s)
+      while true do
+        @logger.debug("SCACSVFlushWatchdog - Time.now = " + Time.now.to_s + " $lastOutputTime=" + @lastOutputTime.to_s + " delay=" + delay.to_s)
+
+        if ( (Time.now.to_i >= (@lastOutputTime.to_i + delay.to_i)) and (@recordCount > 0)) then
+          @logger.debug("SCACSVFlushWatchdog - closeAndRenameCurrentFile")
+          closeAndRenameCurrentFile
+        end
+        @logger.debug("SCACSVFlushWatchdog - Sleeping")
+        sleep 1
+      end
+    end
+  end
+
+  public
+  def receive(event)
+    return unless output?(event)
+
+    @logger.debug("in SCACSV receive")
+
+    if (event['SCAWindowMarker']) 
+      # just eat the marker - don't output it
+      # if we had at least one record output, then close the file and move on 
+      if @recordCount >= 1
+        closeAndRenameCurrentFile
+      end
+    else
+      @formattedPath = event.sprintf(@path)
+      fd = open(@formattedPath)
+      @logger.debug("SCACSVreceive - after opening fd=" + fd.to_s)
+
+      if @recordCount == 0
+        # output header on first line - note, need a minimum of one record for sensible output
+        if @header then 
+#         csv_header = @fields.map { |name| name }
+          fd.write(@header.to_csv(@csv_options))
+        else     
+          fd.write(@fields.to_csv(@csv_options))
+        end
+      end
+
+      csv_values = @fields.map {|name| get_value(name, event)}
+      fd.write(csv_values.to_csv(@csv_options))
+
+      flush(fd)
+      close_stale_files
+
+      # remember state
+      @recordCount = @recordCount + 1
+      @lastOutputTime = Time.now
+
+      # capture the earliest - assumption is that records are in order
+      if (@recordCount) == 1 
+        @startTime = event[@time_field]
+      end
+
+      # for every record, update endTime - again, assumption is that records are in order
+      @endTime = event[@time_field]
+
+      if ((@max_size > 0) and (@recordCount >= max_size))
+        # Have enough records, close it out
+        closeAndRenameCurrentFile
+      end
+
+    end  
+
+  end #def receive
+
+  private
+  def get_value(name, event)
+    val = event[name]
+    case val
+      when Hash
+        return val.to_json
+      else
+        return val
+    end
+  end
+
+  private
+  def epochAsJavaDate( epochTimestamp )
+
+    x = 0
+     if epochTimestamp.to_s.length == 13
+       x = java.util.Date.new(epochTimestamp.to_i)
+     else
+       # should be 10
+       x = java.util.Date.new(epochTimestamp.to_i * 1000)
+     end
+    x
+  end
+
+  def formatOutputTime( timestamp, time_field_format, timestamp_output_format, missingString )
+
+    outputString = ""
+
+    begin
+
+      if timestamp.nil? then
+        @logger.debug("SCACSV " + missingString + " for  #{group}")
+      elsif timestamp_output_format == "epoch" then  
+        outputString = timestamp.to_s  
+      elsif timestamp_output_format == "" then
+        # use time_field format
+        if time_field_format == "epoch" then
+          outputString = timestamp.to_s
+        else
+          df = java.text.SimpleDateFormat.new(time_field_format)
+          outputString = df.format(epochAsJavaDate(timestamp))
+        end
+      else # explicit java timeformat supplied
+        df = java.text.SimpleDateFormat.new(timestamp_output_format)
+        outputString = df.format(epochAsJavaDate(timestamp))
+      end
+ 
+    rescue Exception => e
+      @logger.error("Exception determining output file timestamp. " + missingString, :exception => e)
+      outputString = missingString
+    end
+
+    outputString
+
+  end
+
+  def closeAndRenameCurrentFile
+
+    # cloned and changed from the 'file.rb' operator
+    # even though this is in a loop - assumption is that we have one file here for the SCA CSV use
+    @files.each do |path, fd|
+      begin
+        fd.close
+        @files.delete(path) # so it will be forgotten and we can open it up again if needed
+        @logger.debug("closeAndRenameCurrentFile #{path}", :fd => fd)
+
+        # Now the various time adjustments
+
+        begin # determine start&end times
+
+          if (@time_field_format != "epoch")
+            # if not epoch, then we expect java timestamp format
+            # so must convert start/end times
+
+            df = java.text.SimpleDateFormat.new(@time_field_format)
+            nStartTime = df.parse(@startTime)
+            nEndTime   = df.parse(@endTime)
+
+            @startTime = df.parse(@startTime).getTime
+            @endTime   = df.parse(@endTime).getTime
+
+          end
+
+          # Ensure epoch time from here on out
+
+          if (!@startTime.nil?)
+            @startTime = @startTime.to_i + @tz_offset
+          end
+
+          if (!@endTime.nil?)
+            @endTime   = @endTime.to_i + @tz_offset
+            if (@increment_time)
+              # increment is used to ensure that the end-time on the filename is after the last data value
+
+              @endTime = @endTime.to_i + 1000 # 1000ms = 1sec
+
+            end
+          end
+
+          # then do conversion for output
+
+#        @startTime = formatOutputTime( time, time_field_format, timestamp_output_format, missingString )
+         @startTime = formatOutputTime( @startTime, @time_field_format, @timestamp_output_format, "noStartTime" )
+         @endTime   = formatOutputTime( @endTime,   @time_field_format, @timestamp_output_format, "noEndTime" )
+         
+        rescue Exception => e
+          @logger.error("Exception while flushing and closing files - preparing start/end time", :exception => e)
+          raise
+        end
+
+        # timestamps are strings here
+
+        newFilename = "#{group}" + "__" + @startTime + "__" + @endTime + ".csv"
+
+        if newFilename.include? '/'
+          @logger.error("New filename " + newFilename + " cannot contain / characters. Check the timestamp format. / characters stripped from filename")
+          newFilename = newFilename.delete! '/'
+        end
+
+        realdirpath = File.dirname(File.realdirpath("#{path}"))
+        realdirpath = File.dirname(File.realdirpath(path))
+        oldFilename = File.basename(path)
+
+        File.rename(realdirpath + "/" + oldFilename, realdirpath + "/" + newFilename)
+        
+        # reset record count so we'll pick up new start time, and put a header on next file 
+        # when a new record comes in
+        @recordCount = 0
+        @lastOutputTime = Time.now
+
+      rescue Exception => e
+        @logger.error("Exception while flushing and closing files.", :exception => e)
+      end
+    end
+
+  end
+
+  def teardown
+     @logger.debug("SCACSV - Teardown: closing files")
+
+    Thread.kill(@timerThread)
+    closeAndRenameCurrentFile
+
+    finished
+  end
+
+end # class LogStash::Outputs::SCACSV
+

data/logstash-output-scacsv.gemspec

@@ -0,0 +1,24 @@
+Gem::Specification.new do |s|
+  s.name = 'logstash-output-scacsv'
+  s.version         = "1.0.0"
+  s.licenses = ["Apache License (2.0)"]
+  s.summary = "Receives a stream of events and outputs files meeting the csv reqmts for IBM SmartCloudAnalytics Predictive Insights"
+  s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
+  s.authors = ["Robert Mckeown"]
+  s.email = "rmckeown@us.ibm.com"
+  s.homepage = "https://github.com/IBM-ITOAdev/logstash-output-scacsv/blob/master/README.md"
+  s.require_paths = ["lib"]
+
+  # Files
+  s.files = `git ls-files`.split($\)
+   # Tests
+  s.test_files = s.files.grep(%r{^(test|spec|features)/})
+
+  # Special flag to let us know this is actually a logstash plugin
+  s.metadata = { "logstash_plugin" => "true", "logstash_group" => "output" }
+
+  # Gem dependencies
+  s.add_runtime_dependency "logstash-core", ">= 1.4.0", "< 2.0.0"
+  s.add_runtime_dependency "logstash-codec-plain"
+  s.add_development_dependency "logstash-devutils"
+end

data/spec/outputs/scacsv_spec.rb

@@ -0,0 +1,21 @@
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/example"
+require "logstash/codecs/plain"
+require "logstash/event"
+
+describe LogStash::Outputs::Example do
+  let(:sample_event) { LogStash::Event.new }
+  let(:output) { LogStash::Outputs::Example.new }
+
+  before do
+    output.register
+  end
+
+  describe "receive message" do
+    subject { output.receive(sample_event) }
+
+    it "returns a string" do
+      expect(subject).to eq("Event received")
+    end
+  end
+end

metadata

@@ -0,0 +1,100 @@
+--- !ruby/object:Gem::Specification
+name: logstash-output-scacsv
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+platform: ruby
+authors:
+- Robert Mckeown
+autorequire:
+bindir: bin
+cert_chain: []
+date: 2015-06-12 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: logstash-core
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: 1.4.0
+    - - <
+      - !ruby/object:Gem::Version
+        version: 2.0.0
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-codec-plain
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :runtime
+- !ruby/object:Gem::Dependency
+  name: logstash-devutils
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  prerelease: false
+  type: :development
+description: This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program
+email: rmckeown@us.ibm.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- Gemfile
+- README.md
+- Rakefile
+- lib/logstash/outputs/scacsv.rb
+- logstash-output-scacsv.gemspec
+- spec/outputs/scacsv_spec.rb
+homepage: https://github.com/IBM-ITOAdev/logstash-output-scacsv/blob/master/README.md
+licenses:
+- Apache License (2.0)
+metadata:
+  logstash_plugin: 'true'
+  logstash_group: output
+post_install_message:
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project:
+rubygems_version: 2.1.9
+signing_key:
+specification_version: 4
+summary: Receives a stream of events and outputs files meeting the csv reqmts for IBM SmartCloudAnalytics Predictive Insights
+test_files:
+- spec/outputs/scacsv_spec.rb