logstash-lib 1.3.2

data/spec/sqlite-test.rb

@@ -0,0 +1,81 @@
+require "rubygems"
+require "sequel"
+require "jdbc/sqlite3" 
+
+    public
+    def init_placeholder_table(db)
+      begin
+        db.create_table :since_table do 
+          String :table
+          Int    :place
+        end
+      rescue
+        p 'since tables already exists'
+      end
+    end
+
+    public 
+    def init_placeholder(db, table)
+      p "init placeholder for #{table}"
+      since = db[:since_table]
+      since.insert(:table => table, :place => 1)
+    end
+
+    public
+    def get_placeholder(db, table)
+      since = db[:since_table]
+      x = since.where(:table => "#{table}")
+      p x
+      if x[:place].nil?
+        p 'place is 0'
+        init_placeholder(db, table) 
+        return 0
+      else
+        p "placeholder already exists, it is #{x[:place]}"
+        return x[:place][:place]
+      end
+    end
+
+    public
+    def update_placeholder(db, table, place)
+      since = db[:since_table]
+      since.where(:table => table).update(:place => place)
+    end
+      
+    public 
+    def get_all_tables(db)
+      tables = db["SELECT * FROM sqlite_master WHERE type = 'table'"].map {|table| table[:name]}
+      tables.delete_if { |table| table == 'since_table' }
+      return tables
+    end
+    
+    public
+    def get_n_rows_from_table(db, table, offset, limit)
+      p "Selecting from #{table} where id is at leasat #{offset}"
+      dataset = db["SELECT * FROM #{table}"]
+      return db["SELECT * FROM #{table} WHERE (id >= #{offset}) ORDER BY 'id' LIMIT #{limit}"].map { |row| row }
+    end
+      
+    @DB = Sequel.connect("jdbc:sqlite:/home/ec2-user/u2/log/log.db") 
+
+    tables = get_all_tables(@DB)
+
+    #init table stuff
+    table_data = Hash.new
+    tables.each{ |table|
+      init_placeholder_table(@DB)
+      last_place = get_placeholder(@DB, table)
+      table_data[table] = { :name => table, :place => last_place }
+      #puts table
+    }
+
+    #looped tabled stuff
+    table_data.each{ |k, table|
+      puts table
+      offset = table[:place]
+      limit = 5
+      table_name = table[:name]
+      puts get_n_rows_from_table(@DB, table_name, offset, limit)
+      update_placeholder(@DB, table_name, offset+limit)
+    }
+

data/spec/support/LOGSTASH-733.rb

@@ -0,0 +1,21 @@
+# This spec covers the question here:
+# https://logstash.jira.com/browse/LOGSTASH-733
+
+require "test_utils"
+
+describe "LOGSTASH-733" do
+  extend LogStash::RSpec
+  describe  "pipe-delimited fields" do
+    config <<-CONFIG
+      filter {
+        kv { field_split => "|" }
+      }
+    CONFIG
+
+    sample "field1=test|field2=another test|field3=test3" do
+      insist { subject["field1"] } == "test"
+      insist { subject["field2"] } == "another test"
+      insist { subject["field3"] } == "test3"
+    end
+  end
+end

data/spec/support/LOGSTASH-820.rb

@@ -0,0 +1,25 @@
+# encoding: utf-8
+# This spec covers the question here:
+# https://logstash.jira.com/browse/LOGSTASH-820
+
+require "test_utils"
+
+describe "LOGSTASH-820" do
+  extend LogStash::RSpec
+  describe  "grok with unicode" do
+    config <<-CONFIG
+      filter {
+        grok {
+          #pattern => "<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"
+          pattern => "<%{POSINT:syslog_pri}>%{SPACE}%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{PROG:syslog_program}(:?)(?:\\[%{GREEDYDATA:syslog_pid}\\])?(:?) %{GREEDYDATA:syslog_message}"
+        }
+      }
+    CONFIG
+
+    sample "<22>Jan  4 07:50:46 mailmaster postfix/policy-spf[9454]: : SPF permerror (Junk encountered in record 'v=spf1 mx a:mail.domain.no ip4:192.168.0.4 �all'): Envelope-from: email@domain.no" do
+      insist { subject["tags"] }.nil?
+      insist { subject["syslog_pri"] } == "22"
+      insist { subject["syslog_program"] } == "postfix/policy-spf"
+    end
+  end
+end

data/spec/support/akamai-grok.rb

@@ -0,0 +1,26 @@
+require "test_utils"
+
+describe "..." do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grok {
+        pattern => "%{COMBINEDAPACHELOG}"
+      }
+     
+     
+      date {
+        # Try to pull the timestamp from the 'timestamp' field
+        match => [ "timestamp", "dd'/'MMM'/'yyyy:HH:mm:ss Z" ]
+      }
+    }
+  CONFIG
+
+  line = '192.168.1.1 - - [25/Mar/2013:20:33:56 +0000] "GET /www.somewebsite.co.uk/dwr/interface/AjaxNewsletter.js HTTP/1.1" 200 794 "http://www.somewebsite.co.uk/money/index.html" "Mozilla/5.0 (Linux; U; Android 2.3.6; en-gb; GT-I8160 Build/GINGERBREAD) AppleWebKit/533.1 (KHTML, like Gecko) Version/4.0 Mobile Safari/533.1" "NREUM=s=1364243891214&r=589267&p=101913; __utma=259942479.284548354.1358973919.1363109625.1364243485.15; __utmb=259942479.4.10.1364243485; __utmc=259942479; __utmz=259942479.1359409342.3.3.utmcsr=investing.somewebsite.co.uk|utmccn=(referral)|utmcmd=referral|utmcct=/performance/overview/; asi_segs=D05509_10903|D05509_11337|D05509_11335|D05509_11341|D05509_11125|D05509_11301|D05509_11355|D05509_11508|D05509_11624|D05509_10784|D05509_11003|D05509_10699|D05509_11024|D05509_11096|D05509_11466|D05509_11514|D05509_11598|D05509_11599|D05509_11628|D05509_11681; rsi_segs=D05509_10903|D05509_11337|D05509_11335|D05509_11341|D05509_11125|D05509_11301|D05509_11355|D05509_11508|D05509_11624|D05509_10784|D05509_11003|D05509_10699|D05509_11024|D05509_11096|D05509_11466|D05509_11514|D05509_11598|D05509_11599|D05509_11628|D05509_11681|D05509_11701|D05509_11818|D05509_11850|D05509_11892|D05509_11893|D05509_12074|D05509_12091|D05509_12093|D05509_12095|D05509_12136|D05509_12137|D05509_12156|D05509_0; s_pers=%20s_nr%3D1361998955946%7C1364590955946%3B%20s_pn2%3D/money/home%7C1364245284228%3B%20s_c39%3D/money/home%7C1364245522830%3B%20s_visit%3D1%7C1364245693534%3B; s_sess=%20s_pn%3D/money/home%3B%20s_cc%3Dtrue%3B%20s_sq%3D%3B"'
+
+  sample line do
+    #puts subject["@timestamp"]
+    #puts subject["timestamp"]
+  end
+end

data/spec/support/date-http.rb

@@ -0,0 +1,17 @@
+require "test_utils"
+
+describe "http dates", :if => RUBY_ENGINE == "jruby" do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      date {
+        match => [ "timestamp", "dd/MMM/yyyy:HH:mm:ss Z" ]
+      }
+    }
+  CONFIG
+
+  sample("timestamp" => "25/Mar/2013:20:33:56 +0000") do
+    insist { subject["@timestamp"] } == Time.iso8601("2013-03-25T20:33:56.000Z")
+  end
+end

data/spec/support/postwait1.rb

@@ -0,0 +1,26 @@
+require "test_utils"
+
+describe "some stuff" do
+  extend LogStash::RSpec
+
+  config <<-'CONFIG'
+    filter {
+      grep {
+          tags => web
+          drop => false
+          add_field => [ "application", "web" ]
+      }
+     
+      mutate {
+          tags => web
+          #replace => [ "message", "%{request}" ]
+          add_field => [ "message", "%{request}" ]
+      }
+    }
+  CONFIG
+
+  sample("tags" => [ "web" ], "request" => "hello") do
+    insist { subject["tags"] }.include?("web")
+    insist { subject["message"] } == "hello"
+  end
+end

data/spec/support/pull375.rb

@@ -0,0 +1,21 @@
+# This spec covers the question here:
+# https://github.com/logstash/logstash/pull/375
+
+require "test_utils"
+
+describe "pull #375" do
+  extend LogStash::RSpec
+  describe  "kv after grok" do
+    config <<-CONFIG
+      filter {
+        grok { pattern => "%{URIPATH:mypath}%{URIPARAM:myparams}" }
+        kv { source => "myparams" field_split => "&?" }
+      }
+    CONFIG
+
+    sample "/some/path?foo=bar&baz=fizz" do
+      insist { subject["foo"] } == "bar"
+      insist { subject["baz"] } == "fizz"
+    end
+  end
+end

data/spec/test_utils.rb

@@ -0,0 +1,125 @@
+if ENV['COVERAGE']
+  require 'simplecov'
+  require 'coveralls'
+
+  SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[
+    SimpleCov::Formatter::HTMLFormatter,
+    Coveralls::SimpleCov::Formatter
+  ]
+  SimpleCov.start do
+    add_filter 'spec/'
+    add_filter 'vendor/'
+  end
+end
+require "insist"
+require "logstash/agent"
+require "logstash/pipeline"
+require "logstash/event"
+require "logstash/logging"
+require "insist"
+require "stud/try"
+
+$TESTING = true
+if RUBY_VERSION < "1.9.2"
+  $stderr.puts "Ruby 1.9.2 or later is required. (You are running: " + RUBY_VERSION + ")"
+  $stderr.puts "Options for fixing this: "
+  $stderr.puts "  * If doing 'ruby bin/logstash ...' add --1.9 flag to 'ruby'"
+  $stderr.puts "  * If doing 'java -jar ... ' add -Djruby.compat.version=RUBY1_9 to java flags"
+  raise LoadError
+end
+
+$logger = LogStash::Logger.new(STDOUT)
+if ENV["TEST_DEBUG"]
+  $logger.level = :debug
+else
+  $logger.level = :error
+end
+
+module LogStash
+  module RSpec
+    def config(configstr)
+      let(:config) { configstr }
+    end # def config
+
+    def type(default_type)
+      let(:default_type) { default_type }
+    end
+    
+    def tags(*tags)
+      let(:default_tags) { tags }
+      puts "Setting default tags: #{@default_tags}"
+    end
+
+    def sample(sample_event, &block)
+      name = sample_event.is_a?(String) ? sample_event : sample_event.to_json
+      name = name[0..50] + "..." if name.length > 50
+
+      describe "\"#{name}\"" do
+        extend LogStash::RSpec
+        let(:pipeline) { LogStash::Pipeline.new(config) }
+        let(:event) do
+          sample_event = [sample_event] unless sample_event.is_a?(Array)
+          next sample_event.collect do |e|
+            e = { "message" => e } if e.is_a?(String)
+            next LogStash::Event.new(e)
+          end
+        end
+
+        let(:results) do
+          results = []
+          count = 0
+          pipeline.instance_eval { @filters.each(&:register) }
+          event.each do |e|
+            extra = []
+            pipeline.filter(e) do |new_event|
+              extra << new_event
+            end
+            results << e if !e.cancelled?
+            results += extra.reject(&:cancelled?)
+          end
+
+          # TODO(sissel): pipeline flush needs to be implemented.
+          #results += pipeline.flush
+          next results
+        end
+
+        subject { results.length > 1 ? results: results.first }
+
+        it("when processed", &block)
+      end
+    end # def sample
+
+    def input(&block)
+      it "inputs" do
+        pipeline = LogStash::Pipeline.new(config)
+        queue = Queue.new
+        pipeline.instance_eval do 
+          @output_func = lambda { |event| queue << event }
+        end
+        block.call(pipeline, queue)
+        pipeline.shutdown
+      end
+    end # def input
+
+    def agent(&block)
+      require "logstash/pipeline"
+
+      it("agent(#{caller[0].gsub(/ .*/, "")}) runs") do
+        pipeline = LogStash::Pipeline.new(config)
+        pipeline.run
+        block.call
+      end
+    end # def agent
+
+  end # module RSpec
+end # module LogStash
+
+class Shiftback
+  def initialize(&block)
+    @block = block
+  end
+
+  def <<(event)
+    @block.call(event)
+  end
+end # class Shiftback

data/spec/util/fieldeval_spec.rb

@@ -0,0 +1,44 @@
+require "test_utils"
+require "logstash/util/fieldreference"
+
+describe LogStash::Util::FieldReference, :if => true do
+  it "should permit simple key names" do
+    str = "hello"
+    m = eval(subject.compile(str))
+    data = { "hello" => "world" }
+    insist { m.call(data) } == data[str]
+  end
+
+  it "should permit [key][access]" do
+    str = "[hello][world]"
+    m = eval(subject.compile(str))
+    data = { "hello" => { "world" => "foo", "bar" => "baz" } }
+    insist { m.call(data) } == data["hello"]["world"]
+  end
+  it "should permit [key][access]" do
+    str = "[hello][world]"
+    m = eval(subject.compile(str))
+    data = { "hello" => { "world" => "foo", "bar" => "baz" } }
+    insist { m.call(data) } == data["hello"]["world"]
+  end
+  
+  it "should permit blocks" do
+    str = "[hello][world]"
+    code = subject.compile(str)
+    m = eval(subject.compile(str))
+    data = { "hello" => { "world" => "foo", "bar" => "baz" } }
+    m.call(data) { |obj, key| obj.delete(key) }
+
+    # Make sure the "world" key is removed.
+    insist { data["hello"] } == { "bar" => "baz" }
+  end
+
+  it "should permit blocks #2" do
+    str = "simple"
+    code = subject.compile(str)
+    m = eval(subject.compile(str))
+    data = { "simple" => "things" }
+    m.call(data) { |obj, key| obj.delete(key) }
+    insist { data }.empty?
+  end
+end

data/test/jenkins/config.xml.erb

@@ -0,0 +1,74 @@
+<?xml version='1.0' encoding='UTF-8'?>
+<project>
+  <actions/>
+  <description></description>
+  <keepDependencies>false</keepDependencies>
+  <properties/>
+  <scm class="hudson.plugins.git.GitSCM" plugin="git@1.1.25">
+    <configVersion>2</configVersion>
+    <userRemoteConfigs>
+      <hudson.plugins.git.UserRemoteConfig>
+        <name></name>
+        <refspec></refspec>
+        <url>https://github.com/logstash/logstash.git</url>
+      </hudson.plugins.git.UserRemoteConfig>
+    </userRemoteConfigs>
+    <branches>
+      <hudson.plugins.git.BranchSpec>
+        <name>master</name>
+      </hudson.plugins.git.BranchSpec>
+    </branches>
+    <disableSubmodules>false</disableSubmodules>
+    <recursiveSubmodules>false</recursiveSubmodules>
+    <doGenerateSubmoduleConfigurations>false</doGenerateSubmoduleConfigurations>
+    <authorOrCommitter>false</authorOrCommitter>
+    <clean>false</clean>
+    <wipeOutWorkspace>false</wipeOutWorkspace>
+    <pruneBranches>false</pruneBranches>
+    <remotePoll>false</remotePoll>
+    <ignoreNotifyCommit>false</ignoreNotifyCommit>
+    <useShallowClone>false</useShallowClone>
+    <buildChooser class="hudson.plugins.git.util.DefaultBuildChooser"/>
+    <gitTool>Default</gitTool>
+    <submoduleCfg class="list"/>
+    <relativeTargetDir></relativeTargetDir>
+    <reference></reference>
+    <excludedRegions></excludedRegions>
+    <excludedUsers></excludedUsers>
+    <gitConfigName></gitConfigName>
+    <gitConfigEmail></gitConfigEmail>
+    <skipTag>false</skipTag>
+    <includedRegions></includedRegions>
+    <scmName></scmName>
+  </scm>
+  <canRoam>true</canRoam>
+  <disabled>false</disabled>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <triggers class="vector">
+    <com.cloudbees.jenkins.GitHubPushTrigger plugin="github@1.4">
+      <spec></spec>
+    </com.cloudbees.jenkins.GitHubPushTrigger>
+  </triggers>
+  <concurrentBuild>false</concurrentBuild>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>export JRUBY_OPTS=--1.9
+
+bundle install
+bundle exec rspec spec/<%= plugin_path %>
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers>
+    <ruby-proxy-object>
+      <ruby-object ruby-class="Jenkins::Plugin::Proxies::BuildWrapper" pluginid="rvm">
+        <pluginid pluginid="rvm" ruby-class="String">rvm</pluginid>
+        <object ruby-class="RvmWrapper" pluginid="rvm">
+          <impl pluginid="rvm" ruby-class="String">1.6.8@logstash</impl>
+        </object>
+      </ruby-object>
+    </ruby-proxy-object>
+  </buildWrappers>
+</project>