logstash-input-mongodb 0.1.1 → 0.1.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 797d63879b842bd324be17f4afafeb142cbc16d4
-  data.tar.gz: 8becb42077309b41d88a2b1808d00980433af012
+  metadata.gz: 82148f9ed66d81ea6da480e90ed78600c6a92e42
+  data.tar.gz: 328b9ec0aa5f3de550f70d37b84063dbbf85fb15
 SHA512:
-  metadata.gz: 9b0f0be58b4a1b57cd24d8560e5bb71649bf93775e33bc570c94ac30a39c2fecc2baa5e88f00e80058af69dc89594835ec90747fbe9cc196043337fd89b3043b
-  data.tar.gz: e97852fee59bba3b14739fef7a89f6fcde0e9f29a3eede187d006688c68f178ebe668b6c144cff1e4fa72ff4a82ab213835d7e19446bf557006b51eeda41219d
+  metadata.gz: a6d157dbd02e4c914a9c3e4caf02ae5bd281b92f233456c8fcb575b8fd01629235c606d4054cc357c060ff743773378f7f5b816da1943926430ee5a7c100c83d
+  data.tar.gz: 00510814687f7a723260e436b9c5396d9148730ba81f324aee5d6cf859d43846fb9893a38d4fb1bfb6034df8322dd889cca7c77c171d4d2668964587e1162825

data/Gemfile

@@ -1,3 +1,2 @@
 source 'https://rubygems.org'
 gemspec
-gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"

data/Gemfile.lock

@@ -0,0 +1,106 @@
+PATH
+  remote: .
+  specs:
+    logstash-input-mongodb (0.1.1)
+      jdbc-sqlite3 (= 3.8.10.1)
+      logstash-codec-plain (~> 0)
+      logstash-core (>= 1.4.0, < 2.0.0)
+      mongo (>= 1.12.2, < 2.0.0)
+      sequel (~> 0)
+      stud (~> 0)
+
+GEM
+  remote: https://rubygems.org/
+  specs:
+    ParseTree (3.0.9)
+      RubyInline (~> 3.9.0)
+      sexp_processor (~> 3.2.0)
+    RubyInline (3.9.0)
+      ZenTest (~> 4.3)
+    ZenTest (4.11.0)
+    bson (1.12.3-java)
+    cabin (0.7.1)
+    clamp (0.6.5)
+    coderay (1.1.0)
+    diff-lcs (1.2.5)
+    ffi (1.9.10-java)
+    filesize (0.0.4)
+    gem_publisher (1.5.0)
+    gems (0.8.3)
+    i18n (0.6.9)
+    insist (1.0.0)
+    jdbc-sqlite3 (3.8.10.1)
+    jrjackson (0.2.9)
+    kramdown (1.8.0)
+    logstash-codec-plain (0.1.6)
+      logstash-core (>= 1.4.0, < 2.0.0)
+    logstash-core (1.5.3-java)
+      cabin (~> 0.7.0)
+      clamp (~> 0.6.5)
+      filesize (= 0.0.4)
+      gems (~> 0.8.3)
+      i18n (= 0.6.9)
+      jrjackson (~> 0.2.9)
+      minitar (~> 0.5.4)
+      pry (~> 0.10.1)
+      stud (~> 0.0.19)
+      thread_safe (~> 0.3.5)
+      treetop (< 1.5.0)
+    logstash-devutils (0.0.15-java)
+      gem_publisher
+      insist (= 1.0.0)
+      kramdown
+      minitar
+      rake
+      rspec (~> 3.1.0)
+      stud (>= 0.0.20)
+    metaid (1.0)
+    method_source (0.8.2)
+    minitar (0.5.4)
+    mongo (1.12.3)
+      bson (= 1.12.3)
+    polyglot (0.3.5)
+    pry (0.10.1-java)
+      coderay (~> 1.1.0)
+      method_source (~> 0.8.1)
+      slop (~> 3.4)
+      spoon (~> 0.0)
+    rake (10.4.2)
+    rspec (3.1.0)
+      rspec-core (~> 3.1.0)
+      rspec-expectations (~> 3.1.0)
+      rspec-mocks (~> 3.1.0)
+    rspec-core (3.1.7)
+      rspec-support (~> 3.1.0)
+    rspec-expectations (3.1.2)
+      diff-lcs (>= 1.2.0, < 2.0)
+      rspec-support (~> 3.1.0)
+    rspec-mocks (3.1.3)
+      rspec-support (~> 3.1.0)
+    rspec-support (3.1.2)
+    ruby2ruby (1.3.1)
+      ruby_parser (~> 2.0)
+      sexp_processor (~> 3.0)
+    ruby_parser (2.3.1)
+      sexp_processor (~> 3.0)
+    sequel (0.5.0.2)
+      ParseTree (>= 2.1.1)
+      RubyInline (>= 3.6.6)
+      metaid
+      ruby2ruby
+    sexp_processor (3.2.0)
+    slop (3.6.0)
+    spoon (0.0.4)
+      ffi
+    stud (0.0.20)
+    thread_safe (0.3.5-java)
+    treetop (1.4.15)
+      polyglot
+      polyglot (>= 0.3.1)
+
+PLATFORMS
+  java
+
+DEPENDENCIES
+  logstash-devutils (~> 0)
+  logstash-input-mongodb!

data/README.md

@@ -6,90 +6,31 @@ It is fully free and fully open source. The license is Apache 2.0, meaning you a
 
 ## Documentation
 
-Logstash provides infrastructure to automatically generate documentation for this plugin. We use the asciidoc format to write documentation so any comments in the source code will be first converted into asciidoc and then into html. All plugin documentation are placed under one [central location](http://www.elasticsearch.org/guide/en/logstash/current/).
-
-- For formatting code or config example, you can use the asciidoc `[source,ruby]` directive
-- For more asciidoc formatting tips, see the excellent reference here https://github.com/elasticsearch/docs#asciidoc-guide
-
-## Need Help?
-
-Need help? Try #logstash on freenode IRC or the logstash-users@googlegroups.com mailing list.
-
-## Developing
-
-### 1. Plugin Developement and Testing
-
-#### Code
-- To get started, you'll need JRuby with the Bundler gem installed.
-
-- Create a new plugin or clone and existing from the GitHub [logstash-plugins](https://github.com/logstash-plugins) organization.
-
-- Install dependencies
-```sh
-bundle install
+### Configuration
+
+Example
+```
+input {
+  mongodb {
+    uri => 'mongodb://10.0.0.30/my-logs?ssl=true'
+    path => '/opt/logstash-mongodb/logstash_sqlite.db'
+    collection => 'events_'
+    unpack_mongo_id => true
+    batch_size => 5000
+  }
+}
+
+filter {
+  date {
+    match => [ "logdate", "ISO8601" ]
+  }
+}
+
+output {
+  redis {
+    host => "localhost"
+    data_type => "list"
+    key => "logstash-mylogs"
+  }
+}
 ```
-
-#### Test
-
-```sh
-bundle exec rspec
-```
-
-The Logstash code required to run the tests/specs is specified in the `Gemfile` by the line similar to:
-```ruby
-gem "logstash", :github => "elasticsearch/logstash", :branch => "1.5"
-```
-To test against another version or a local Logstash, edit the `Gemfile` to specify an alternative location, for example:
-```ruby
-gem "logstash", :github => "elasticsearch/logstash", :ref => "master"
-```
-```ruby
-gem "logstash", :path => "/your/local/logstash"
-```
-
-Then update your dependencies and run your tests:
-
-```sh
-bundle install
-bundle exec rspec
-```
-
-### 2. Running your unpublished Plugin in Logstash
-
-#### 2.1 Run in a local Logstash clone
-
-- Edit Logstash `tools/Gemfile` and add the local plugin path, for example:
-```ruby
-gem "logstash-filter-awesome", :path => "/your/local/logstash-filter-awesome"
-```
-- Update Logstash dependencies
-```sh
-rake vendor:gems
-```
-- Run Logstash with your plugin
-```sh
-bin/logstash -e 'filter {awesome {}}'
-```
-At this point any modifications to the plugin code will be applied to this local Logstash setup. After modifying the plugin, simply rerun Logstash.
-
-#### 2.2 Run in an installed Logstash
-
-- Build your plugin gem
-```sh
-gem build logstash-filter-awesome.gemspec
-```
-- Install the plugin from the Logstash home
-```sh
-bin/plugin install /your/local/plugin/logstash-filter-awesome.gem
-```
-- Start Logstash and proceed to test the plugin
-
-## Contributing
-
-All contributions are welcome: ideas, patches, documentation, bug reports, complaints, and even something you drew up on a napkin.
-
-Programming is not a required skill. Whatever you've seen about open source and maintainers or community members  saying "send patches or die" - you will not see that here.
-
-It is more important to me that you are able to contribute.
-
-For more information about contributing, see the [CONTRIBUTING](https://github.com/elasticsearch/logstash/blob/master/CONTRIBUTING.md) file.

data/lib/logstash/inputs/mongodb.rb

@@ -5,11 +5,9 @@ require "logstash/timestamp"
 require "stud/interval"
 require "socket" # for Socket.gethostname
 require "json"
-require "bson"
+require "mongo"
 
-# Generate a repeating message.
-#
-# This plugin is intented only as an example.
+include Mongo
 
 class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
   config_name "mongodb"
@@ -84,7 +82,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     @logger.debug("init placeholder for #{since_table}_#{mongo_collection_name}")
     since = sqlitedb[SINCE_TABLE]
     mongo_collection = mongodb.collection(mongo_collection_name)
-    first_entry = mongo_collection.find({}, :sort => ['_id', Mongo::ASCENDING], :limit => 1).first
+    first_entry = mongo_collection.find({}).sort('_id' => 1).limit(1).first
     first_entry_id = first_entry['_id'].to_s
     since.insert(:table => "#{since_table}_#{mongo_collection_name}", :place => first_entry_id)
     return first_entry_id
@@ -152,24 +150,29 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
 
   public
   def register
-    require "mongo"
     require "jdbc/sqlite3"
     require "sequel"
-    uriParsed = Mongo::URIParser.new(@uri)
-    conn = uriParsed.connection({})
-    if uriParsed.auths.length > 0
-      uriParsed.auths.each do |auth|
+    mongo_uri = Mongo::URI.new(@uri)
+    hosts_array = mongo_uri.servers
+    db_name = mongo_uri.database
+    ssl_enabled = mongo_uri.options[:ssl]
+    conn = Mongo::Client.new(hosts_array, ssl: ssl_enabled, database: db_name)
+
+    if @db_auths
+      @db_auths.each do |auth|
         if !auth['db_name'].nil?
           conn.add_auth(auth['db_name'], auth['username'], auth['password'], nil)
         end
       end
       conn.apply_saved_authentication()
     end
+
     @host = Socket.gethostname
     @logger.info("Registering MongoDB input", :database => @path)
-    #@mongodb = conn.db(@database)
-    @mongodb = conn.db(uriParsed.db_name)
+
+    @mongodb = conn.database
     @sqlitedb = Sequel.connect("jdbc:sqlite:#{@path}")
+
     # Should check to see if there are new matching tables at a predefined interval or on some trigger
     @collection_data = update_watched_collections(@mongodb, @collection, @sqlitedb)
   end # def register
@@ -186,6 +189,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
 
   def flatten(my_hash)
     new_hash = {}
+    @logger.debug("Raw Hash: #{my_hash}")
     if my_hash.respond_to? :each
       my_hash.each do |k1,v1|
         if v1.is_a?(Hash)
@@ -209,6 +213,7 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
     else
       @logger.debug("Flatten [ERROR]: hash did not respond to :each")
     end
+    @logger.debug("Flattened Hash: #{new_hash}")
     return new_hash
   end
 
@@ -244,7 +249,8 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
             #@logger.debug("EVENT looks like: "+event.to_s)
             # Extract the HOST_ID and PID from the MongoDB BSON::ObjectID
             if @unpack_mongo_id
-              doc_obj_bin = doc['_id'].to_a.pack("C*").unpack("a4 a3 a2 a3")
+              doc_hex_bytes = doc['_id'].to_s.each_char.each_slice(2).map {|b| b.join.to_i(16) }
+              doc_obj_bin = doc_hex_bytes.pack("C*").unpack("a4 a3 a2 a3")
               host_id = doc_obj_bin[1].unpack("S")
               process_id = doc_obj_bin[2].unpack("S")
               event['host_id'] = host_id.first.to_i
@@ -265,10 +271,17 @@ class LogStash::Inputs::MongoDB < LogStash::Inputs::Base
 
               flat_doc.each do |k,v|
                 # Check for an integer
-                if /\A[-+]?\d+[.][\d]+\z/ === v
-                  event[k.to_s] = v.to_f
-                elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
-                  event[k.to_s] = v.to_i
+                @logger.debug("key: #{k.to_s} value: #{v.to_s}")
+                if v.is_a? Numeric
+                  event[k.to_s] = v
+                elsif v.is_a? String
+                  if v == "NaN"
+                    event[k.to_s] = Float::NAN
+                  elsif /\A[-+]?\d+[.][\d]+\z/ == v
+                    event[k.to_s] = v.to_f
+                  elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
+                    event[k.to_s] = v.to_i
+                  end
                 else
                   event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
                   if (k.to_s == "tags") && (v.is_a? Array)

data/logstash-input-mongodb.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-input-mongodb'
-  s.version = '0.1.1'
+  s.version = '0.1.3'
   s.licenses = ['Apache License (2.0)']
   s.summary = "This takes entries from mongodb as an input to logstash."
   s.description = "This gem is a logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/plugin install gemname. This gem is not a stand-alone program"
@@ -18,10 +18,11 @@ Gem::Specification.new do |s|
   s.metadata = { "logstash_plugin" => "true", "logstash_group" => "input" }
 
   # Gem dependencies
-  s.add_runtime_dependency 'logstash', '>= 1.4.0', '< 2.0.0'
+  s.add_runtime_dependency 'logstash-core'
   s.add_runtime_dependency 'logstash-codec-plain'
   s.add_runtime_dependency 'stud'
-  s.add_runtime_dependency 'jdbc/sqlite3'
+  s.add_runtime_dependency 'jdbc-sqlite3', '3.8.10.1'
   s.add_runtime_dependency 'sequel'
+  s.add_runtime_dependency 'mongo', '>= 2.0.0'
   s.add_development_dependency 'logstash-devutils'
 end

data/test/flattener_test.rb

@@ -0,0 +1,66 @@
+require 'minitest/autorun'
+require 'json'
+require 'mongo'
+require 'bson'
+
+class TestThis < MiniTest::Test
+  def test_that_this_works
+    input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
+    expected = "your mom"
+    assert_equal flatten(input), expected
+  end
+  def test_the_flat_parser
+    input = {"_id"=>BSON::ObjectId('558de77ec5ed007567574a58'), "tags"=>["http"], "info"=>{"method"=>"POST", "rtime"=>"127", "url"=>"apiginvoice", "data"=>{"a"=>291.35, "b"=>291.25, "c"=>291.16, "d"=>289.68, "e"=>261.73, "f"=>Float::NAN}, "host"=>"bitpay.com", "status"=>200, "referrer"=>nil, "raddr"=>"31.192.114.250", "ver"=>"1.1", "ua"=>nil, "rlen"=>nil, "rlocation"=>nil, "query"=>{}}}
+    expected = "your mom"
+    assert_equal flat_doc(flatten(input)), expected
+  end
+end
+
+def flatten(my_hash)
+  new_hash = {}
+  if my_hash.respond_to? :each
+    my_hash.each do |k1,v1|
+      if v1.is_a?(Hash)
+        v1.each do |k2,v2|
+          if v2.is_a?(Hash)
+            # puts "Found a nested hash"
+            result = flatten(v2)
+            result.each do |k3,v3|
+              new_hash[k1.to_s+"_"+k2.to_s+"_"+k3.to_s] = v3
+            end
+            # puts "result: "+result.to_s+" k2: "+k2.to_s+" v2: "+v2.to_s
+          else
+            new_hash[k1.to_s+"_"+k2.to_s] = v2
+          end
+        end
+      else
+        # puts "Key: "+k1.to_s+" is not a hash"
+        new_hash[k1.to_s] = v1
+      end
+    end
+  end
+  return new_hash
+end
+
+def flat_doc(flat_doc)
+  event = {}
+  flat_doc.each do |k,v|
+    # Check for an integer
+    if v.is_a? Numeric
+      event[k.to_s] = v
+    elsif v.is_a? String
+      if v == "NaN"
+        event[k.to_s] = Float::NAN
+      elsif /\A[-+]?\d+[.][\d]+\z/ == v
+        event[k.to_s] = v.to_f
+      elsif (/\A[-+]?\d+\z/ === v) || (v.is_a? Integer)
+        event[k.to_s] = v.to_i
+      end
+    else
+      event[k.to_s] = v.to_s unless k.to_s == "_id" || k.to_s == "tags"
+      if (k.to_s == "tags") && (v.is_a? Array)
+        event['tags'] = v
+      end
+    end
+  end
+end

metadata

@@ -1,35 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: logstash-input-mongodb
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.3
 platform: ruby
 authors:
 - Philip Hutchins
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-30 00:00:00.000000000 Z
+date: 2015-08-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: logstash
+  name: logstash-core
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: 1.4.0
-    - - "<"
-      - !ruby/object:Gem::Version
-        version: 2.0.0
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: logstash-codec-plain
   requirement: !ruby/object:Gem::Requirement
@@ -59,7 +53,21 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: jdbc/sqlite3
+  name: jdbc-sqlite3
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.8.10.1
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '='
+      - !ruby/object:Gem::Version
+        version: 3.8.10.1
+- !ruby/object:Gem::Dependency
+  name: sequel
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
@@ -73,19 +81,19 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: sequel
+  name: mongo
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
-        version: '0'
+        version: 2.0.0
 - !ruby/object:Gem::Dependency
   name: logstash-devutils
   requirement: !ruby/object:Gem::Requirement
@@ -110,12 +118,14 @@ extra_rdoc_files: []
 files:
 - DEVELOPER.md
 - Gemfile
+- Gemfile.lock
 - LICENSE
 - README.md
 - Rakefile
 - lib/logstash/inputs/mongodb.rb
 - logstash-input-mongodb.gemspec
 - spec/inputs/example_spec.rb
+- test/flattener_test.rb
 homepage: http://www.phutchins.com
 licenses:
 - Apache License (2.0)
@@ -138,9 +148,10 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5
+rubygems_version: 2.4.6
 signing_key: 
 specification_version: 4
 summary: This takes entries from mongodb as an input to logstash.
 test_files:
 - spec/inputs/example_spec.rb
+- test/flattener_test.rb